Hadoop 集成 hive 碰到的版本问题

最近在随大流搞下 hadoop， 太多坑了。前辈们估计跟我差不太多。

把今天的一个版本冲突的问题写到博客里，如果碰到一样的问题。不用走弯路吧。

hadoop 2.2.0 与 hive 0.12  集成时碰到这种异常

 

org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'wordcountJob': Invocation of init method failed; nested exception is java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$AppendRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet;

        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1486)

        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:524)

        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:461)

        at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:295)

        at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223)

        at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:292)

        at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194)

        at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:589)

        at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932)

        at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479)

        at org.springframework.web.context.ContextLoader.configureAndRefreshWebApplicationContext(ContextLoader.java:383)

        at org.springframework.web.context.ContextLoader.initWebApplicationContext(ContextLoader.java:283)

        at org.springframework.web.context.ContextLoaderListener.contextInitialized(ContextLoaderListener.java:112)

        at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:3934)

        at org.apache.catalina.core.StandardContext.start(StandardContext.java:4429)

        at org.apache.catalina.core.ContainerBase.start(ContainerBase.java:1045)

        at org.apache.catalina.core.StandardHost.start(StandardHost.java:722)

        at org.apache.catalina.core.ContainerBase.start(ContainerBase.java:1045)

        at org.apache.catalina.core.StandardEngine.start(StandardEngine.java:443)

        at org.apache.catalina.core.StandardService.start(StandardService.java:516)

        at org.apache.catalina.core.StandardServer.start(StandardServer.java:710)

        at org.apache.catalina.startup.Catalina.start(Catalina.java:583)

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:288)

        at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:413)

Caused by: java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$AppendRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet;

        at java.lang.ClassLoader.defineClass1(Native Method)

        at java.lang.ClassLoader.defineClassCond(ClassLoader.java:631)

        at java.lang.ClassLoader.defineClass(ClassLoader.java:615)

        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:141)

        at org.apache.catalina.loader.WebappClassLoader.findClassInternal(WebappClassLoader.java:1850)

        at org.apache.catalina.loader.WebappClassLoader.findClass(WebappClassLoader.java:890)

        at org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1354)

        at org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1233)

        at java.lang.Class.getDeclaredMethods0(Native Method)

        at java.lang.Class.privateGetDeclaredMethods(Class.java:2427)

        at java.lang.Class.privateGetPublicMethods(Class.java:2547)

        at java.lang.Class.privateGetPublicMethods(Class.java:2557)

        at java.lang.Class.getMethods(Class.java:1410)

        at sun.misc.ProxyGenerator.generateClassFile(ProxyGenerator.java:409)

        at sun.misc.ProxyGenerator.generateProxyClass(ProxyGenerator.java:306)

        at java.lang.reflect.Proxy.getProxyClass(Proxy.java:501)

        at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:581)

        at org.apache.hadoop.ipc.ProtobufRpcEngine.getProxy(ProtobufRpcEngine.java:92)

        at org.apache.hadoop.ipc.RPC.getProtocolProxy(RPC.java:537)

        at org.apache.hadoop.hdfs.NameNodeProxies.createNNProxyWithClientProtocol(NameNodeProxies.java:328)

        at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:235)

        at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:139)

        at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:510)

        at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:453)

        at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:136)

        at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2433)

        at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:88)

        at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2467)

        at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2449)

        at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:367)

        at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:166)

        at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:351)

        at org.apache.hadoop.fs.Path.getFileSystem(Path.java:287)

        at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(FileInputFormat.java:466)

        at org.springframework.data.hadoop.mapreduce.JobFactoryBean.afterPropertiesSet(JobFactoryBean.java:208)

        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1545)

        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1483)

 

想都不用想。一看就是版本冲突问题。这个也不例外。

在网上找了下资料。就是这个google  protobuf 包的版本不对。升级到protobuf-java-2.5.0.jar 就可以了。本来 2.2 也就是用的这个包。用maven 查了下 整个依赖树 也没发现有其他包冲突。 郁闷了。 最后决定把所有 hadoop 的相关包反编译一下看看。 有没有其他包 把 protobuf 的 class 重复打到其他的包中了。

经过排查 项目中引用了 hive-exec-0.12.0.jar 这个包。 这个集成了 hive 所需要的第三方包。

不引用就 ok . 

 

搞定。 最近有几个版本冲突的问题 很棘手。以后再写