- 浏览: 114774 次
- 性别:
- 来自: 北京
-
文章分类
- 全部博客 (109)
- hive (5)
- web (1)
- spring (7)
- struts2 (1)
- s2sh (2)
- mysql (3)
- hadoop (31)
- hbase (6)
- java (8)
- ubuntu (8)
- pig (2)
- Interview (2)
- zookeeper (1)
- system (1)
- 遥控 (1)
- linux (3)
- myeclipse (2)
- Oracle (1)
- redis (9)
- ibatis (2)
- 架构 (2)
- 解析xml (1)
- autoProxy (0)
- jedis (6)
- http://www.infoq.com/cn/articles/tq-redis-copy-build-scalable-cluster (1)
- xmemcached (1)
- 图片服务器 (1)
- 对象池 (0)
- netty (1)
最新评论
-
laoma102:
已经不好使了,能找到最新的吗
spring官方文档 -
di1984HIT:
不错,。不错~
pig安装
import java.util.*;
public class StringUtil {
/**
* Converts an array of string to a string.
*/
public static String arrayToString(String[] stringArray, String OutputDelimiter) {
int linenum = 0;
StringBuffer sb = new StringBuffer();
for (int i = 0; i < stringArray.length; i++) {
linenum++;
if (linenum == 1) {
sb.append(stringArray[i]);
} else {
sb.append(OutputDelimiter);
sb.append(stringArray[i]);
}
}
return sb.toString();
}
/**
* This method takes an input string and returns the default value if the input string is null.
*
* @param input The string that needs to be checked for null
* @defaultValueIfInputIsNull If the "input" is null then replace it by this default value.
*/
public static String ifNull(String input, String defaultValueIfInputIsNull) {
String output = null;
if (input == null) {
output = defaultValueIfInputIsNull;
} else {
output = input;
}
return output;
}
/**
* This method splits a string by the delimiter and returns the value at the given position
*
* @param str The input string
* @param delimiter The delimiter that separates different values of the string
* @param tokenNumber The value of the string at the given tokenNumber. It works with 0 index.
* @return Returns null if the tokenNumber is greater than the total number of tokens in the string
* <p/>
* <b>Example:</b> For string "a:b:c:d:e:f:g", if the method is run with delimiter=":" and tokenNumber=1, it
* will return "b"
*/
public static String getValueAtToken(String str, String delimiter, int tokenNumber) {
String[] tokens = str.split(delimiter, -1);
String returnString = null;
for (int i = 0; i < tokens.length; i++) {
if (tokenNumber == i) {
returnString = tokens[i];
}
}
return returnString;
}
/**
* This method merges 2 input string arrays and returns the merged array as output
*
* @param inputArray1 Input Array 1 that needs to be merged
* @param inputArray2 Input Array 2 that needs to be merged
* @return Returns merged Array that contains all elements of inputArray1 followed by elements of inputArray2
* <p/>
* <b>Example:</b> The following are the input array
* <p/>
* inputArray1 contains elements ("a","b","c")
* <p/>
* inputArray2 contains element ("b","e")
* <p/>
* then the output array contains elements
* <p/>
* ("a","b","c","b","e")
*/
public static String[] mergeArrays(String[] inputArray1, String[] inputArray2) {
String[] outputArray = new String[inputArray1.length + inputArray2.length];
/**
* Copy first array to outputArray
*/
for (int i = 0; i < inputArray1.length; i++) {
outputArray[i] = inputArray1[i];
}
/**
* Copy second array to outputArray
*/
int outputArrayLength = inputArray1.length;
for (int i = 0; i < inputArray2.length; i++) {
outputArray[i + outputArrayLength] = inputArray2[i];
}
return outputArray;
}
/**
* This method returns the dedupe list of elements in the array
* e.g. supose an array has element 1,2,3,4,2,1
* then this method returns
* 1,2,3,4
*
* @param inputArray Input array
* @param outputDelimiter The delimiter used to separate output values
* @return Dedupe list of values in the array
* @throws MalformedLineException
*/
public static String dedupe(String[] inputArray, String outputDelimiter)
throws MalformedLineException {
TrafficEtlHashMap tehm = new TrafficEtlHashMap();
for (int i = 0; i < inputArray.length; i++) {
tehm.put(inputArray[i], "");
}
return tehm.getAllKeys(outputDelimiter);
}
/**
* Same as method dedupe(String[] inputArray, String outputDelimiter) except the input
* is different
*/
public static String dedupe(String inputList, String inputDelimiter, String outputDelimiter)
throws MalformedLineException {
String[] inputArray = inputList.split(inputDelimiter, -1);
return StringUtil.dedupe(inputArray, outputDelimiter);
}
/**
* Get property value from resource bundle
*
* @param rb ResourceBundle instance
* @param propertyName property name
* @return property value
*/
public static String getPropertyValue(ResourceBundle rb, String propertyName) {
String propertyValue = null;
try {
propertyValue = rb.getString(propertyName);
} catch (MissingResourceException ex) {
System.err.println("Could not find value for property: " + propertyName);
}
return propertyValue;
}
/**
* This method parses a string and return true if its an integer else it return false
*
* @param input
* @return
*/
public static boolean isInteger(String input) {
try {
Integer.parseInt(input);
return true;
} catch (Exception e) {
return false;
}
}
}
public class StringUtil {
/**
* Converts an array of string to a string.
*/
public static String arrayToString(String[] stringArray, String OutputDelimiter) {
int linenum = 0;
StringBuffer sb = new StringBuffer();
for (int i = 0; i < stringArray.length; i++) {
linenum++;
if (linenum == 1) {
sb.append(stringArray[i]);
} else {
sb.append(OutputDelimiter);
sb.append(stringArray[i]);
}
}
return sb.toString();
}
/**
* This method takes an input string and returns the default value if the input string is null.
*
* @param input The string that needs to be checked for null
* @defaultValueIfInputIsNull If the "input" is null then replace it by this default value.
*/
public static String ifNull(String input, String defaultValueIfInputIsNull) {
String output = null;
if (input == null) {
output = defaultValueIfInputIsNull;
} else {
output = input;
}
return output;
}
/**
* This method splits a string by the delimiter and returns the value at the given position
*
* @param str The input string
* @param delimiter The delimiter that separates different values of the string
* @param tokenNumber The value of the string at the given tokenNumber. It works with 0 index.
* @return Returns null if the tokenNumber is greater than the total number of tokens in the string
* <p/>
* <b>Example:</b> For string "a:b:c:d:e:f:g", if the method is run with delimiter=":" and tokenNumber=1, it
* will return "b"
*/
public static String getValueAtToken(String str, String delimiter, int tokenNumber) {
String[] tokens = str.split(delimiter, -1);
String returnString = null;
for (int i = 0; i < tokens.length; i++) {
if (tokenNumber == i) {
returnString = tokens[i];
}
}
return returnString;
}
/**
* This method merges 2 input string arrays and returns the merged array as output
*
* @param inputArray1 Input Array 1 that needs to be merged
* @param inputArray2 Input Array 2 that needs to be merged
* @return Returns merged Array that contains all elements of inputArray1 followed by elements of inputArray2
* <p/>
* <b>Example:</b> The following are the input array
* <p/>
* inputArray1 contains elements ("a","b","c")
* <p/>
* inputArray2 contains element ("b","e")
* <p/>
* then the output array contains elements
* <p/>
* ("a","b","c","b","e")
*/
public static String[] mergeArrays(String[] inputArray1, String[] inputArray2) {
String[] outputArray = new String[inputArray1.length + inputArray2.length];
/**
* Copy first array to outputArray
*/
for (int i = 0; i < inputArray1.length; i++) {
outputArray[i] = inputArray1[i];
}
/**
* Copy second array to outputArray
*/
int outputArrayLength = inputArray1.length;
for (int i = 0; i < inputArray2.length; i++) {
outputArray[i + outputArrayLength] = inputArray2[i];
}
return outputArray;
}
/**
* This method returns the dedupe list of elements in the array
* e.g. supose an array has element 1,2,3,4,2,1
* then this method returns
* 1,2,3,4
*
* @param inputArray Input array
* @param outputDelimiter The delimiter used to separate output values
* @return Dedupe list of values in the array
* @throws MalformedLineException
*/
public static String dedupe(String[] inputArray, String outputDelimiter)
throws MalformedLineException {
TrafficEtlHashMap tehm = new TrafficEtlHashMap();
for (int i = 0; i < inputArray.length; i++) {
tehm.put(inputArray[i], "");
}
return tehm.getAllKeys(outputDelimiter);
}
/**
* Same as method dedupe(String[] inputArray, String outputDelimiter) except the input
* is different
*/
public static String dedupe(String inputList, String inputDelimiter, String outputDelimiter)
throws MalformedLineException {
String[] inputArray = inputList.split(inputDelimiter, -1);
return StringUtil.dedupe(inputArray, outputDelimiter);
}
/**
* Get property value from resource bundle
*
* @param rb ResourceBundle instance
* @param propertyName property name
* @return property value
*/
public static String getPropertyValue(ResourceBundle rb, String propertyName) {
String propertyValue = null;
try {
propertyValue = rb.getString(propertyName);
} catch (MissingResourceException ex) {
System.err.println("Could not find value for property: " + propertyName);
}
return propertyValue;
}
/**
* This method parses a string and return true if its an integer else it return false
*
* @param input
* @return
*/
public static boolean isInteger(String input) {
try {
Integer.parseInt(input);
return true;
} catch (Exception e) {
return false;
}
}
}
发表评论
-
mapreduce Bet
2012-04-11 15:00 924import java.io.IOException; imp ... -
hadoop 输出格式
2012-04-05 17:18 729http://blog.csdn.net/dajuezhao/ ... -
hadoop mapreduce 原理
2012-03-31 16:14 699http://www.cnblogs.com/forfutur ... -
hadoop搭建问题
2012-03-30 13:23 810file:///E:/hadoop/搭建/hadoop集群搭建 ... -
hadoop输出文件格式
2012-03-26 10:09 658http://apps.hi.baidu.com/share/ ... -
hadoop 学习
2012-03-26 09:48 658http://hi.baidu.com/shuyan50/bl ... -
hadoop提高性能建议
2012-03-22 22:40 686http://langyu.iteye.com/blog/91 ... -
hadoop例子
2012-03-22 22:09 736http://www.hadoopor.com/thread- ... -
hadoop
2012-04-25 13:16 754精通HADOOP http://blog.csdn.net/ ... -
Hadoop Hive与Hbase整合
2012-03-07 15:02 362http://www.open-open.com/lib/vi ... -
hive hadoop 代码解析
2012-04-25 13:16 798http://www.tbdata.org/archives/ ... -
Hadoop MapReduce操作MySQL
2012-03-05 17:33 895http://www.javabloger.com/artic ... -
hadoop hdfs常用操作类
2012-03-05 10:03 1962import java.io.IOException; ... -
hdfs 操作类自己的
2012-03-02 17:57 562package operateFile; import ... -
hadoo 文件常用操作
2012-03-02 15:53 779http://www.360doc.com/content/1 ... -
Mapper,Reducer,Wrapper的Java模板
2012-03-02 08:24 1118http://www.easyigloo.org/?p=114 ... -
hadoop基础知识
2012-03-02 08:00 744http://www.blogjava.net/killme2 ... -
hadoop 自己封装的接口
2012-04-25 13:16 695http://www.360doc.com/content/1 ... -
HadoopFileUtil
2012-03-01 14:42 1840import java.io.File; import jav ... -
hadoop ExtendedFileUtil
2012-03-01 14:34 1056在Hadoop编写生产环境的任务时,定义以下任务,要求是相同的 ...
相关推荐
Exception in thread "main" java.lang.UnsatisfiedLinkError:''boolean org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(java.lang.String, int)' * at org.apache.hadoop.io.nativeio.NativeIO$...
解决 java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V
这些参数将会作为Java Application的`main`方法中的参数数组`String[] args`接收。 ##### 3.2 获取Java Application的退出状态码 在Shell脚本中,还可以通过`$?`获取上一个命令的退出状态码。对于Java Application...
hadoop-2.6.0的hadoop.dll+winutils.exe 解决java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V
解决Exceptionin thread "main" java.lang.UnsatisfiedLinkError:org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V
hadoop-3.0.0中的hdfs.dll和winutils.exe 解决java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V
解决:java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V 改异常是由于hadoop.dll 版本问题出现的,2.4之前的和之后的需要...
hadoop 在windows开发环境中,运行会出现 org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V .原因是缺失hadoop.dll 文件
### 使用命令行编译打包运行自己的MapReduce程序 Hadoop2.6.0 #### Hadoop 2.x 版本变化及依赖分析 在Hadoop 2.x版本中,相较于早期版本,其架构和依赖库有了明显的变化。在早期版本如1.x中,所有的依赖都集中在`...
hadoop2.2.0 在window下进行...源码org.apache.hadoop.util.Shell.java 的277行fullName的路径如: String fullExeName = "d:/hadoop" + File.separator + executable; 然后把winutils.exe放入到d:/hadoop目录下
import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org....
import java.util.Iterator; public class WordCountReducer extends Reducer, IntWritable, Text, IntWritable> { private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable...
import org.apache.hadoop.util.GenericOptionsParser; public class Dedup { // map阶段将输入中的value复制到输出数据的key public static class TokenizerMapper extends Mapper, Text, Text, IntWritable> { ...
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "word count"); job.setJarByClass(WordCount.class); job....
import org.apache.hadoop.hbase.util.Bytes; public class HBaseExample { public static void main(String[] args) throws Exception { // 创建 HBase 配置 org.apache.hadoop.conf.Configuration config = ...
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "word count"); job.setJarByClass(WordCount.class); job....
import java.util.Scanner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; /** * 定义RPC通讯Client端 */ public class SayRpcClient { private static Scanner sc; ...
### Apache日志Hadoop大数据分析项目:清洗数据核心功能解说及代码实现 #### 一、项目背景 在大数据分析领域,尤其是针对Web服务器的日志分析过程中,原始数据往往杂乱无章,包含大量无关信息。为了提高数据分析的...
String text = value.toString(); Matcher matcher = pattern.matcher(text); if (matcher.find()) { output.collect(key, value); } } } ``` 在这个示例中,`RegMapper`继承自`Mapper`接口,并重写了`map`...
import org.apache.hadoop.util.GenericOptionsParser; public class WordCount { public WordCount() { } public static void main(String[] args) throws Exception { Configuration conf = new ...