Hadoop Client Kerberos Error writing to HDFS from Java SDK - java

I am trying to load data into hdfs from local file system using Java SDK. I have created the jar(with all the dependencies[hadoop-mapreduce-client-common, hadoop-common, hadoop-client ver 2.7.3] Java 1.7) and when I ran it as a hadoop command it is working fine, but when I run it as a java jar it is throwing an error. Keytab is working fine and I can run kinit & klist successfully. Any help on this would be appreciated.
hadoop jar ./AppHadoop-1.0-SNAPSHOT.jar org.apache.hadoop.examples.ProxyCreateSecureHDFSFile /user/hdfs_java user#XXXXX.XXXXX.COM keytabs/user.keytab /apache/hadoop/conf/core-site.xml /apache/hadoop/conf/hdfs-site.xml /apache/hadoop/conf/mapred-site.xml
Working as expected. Success.
java -cp ./AppHadoop-1.0-SNAPSHOT.jar org.apache.hadoop.examples.ProxyCreateSecureHDFSFile /user/hdfs_java user#XXXXX.XXXXX.COM keytabs/user.keytab /apache/hadoop/conf/core-site.xml /apache/hadoop/conf/hdfs-site.xml /apache/hadoop/conf/mapred-site.xml
Output
Home Directory : hdfs://namenode:8020/user/
Working Directory : hdfs://namenode:8020/user/
--------EXCEPTION________________
java.io.IOException: Failed on local exception: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]; Host Details : local host is: *"clientnode"*; destination host is: *"namenode"*:8020;
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:776)
at org.apache.hadoop.ipc.Client.call(Client.java:1479)
at org.apache.hadoop.ipc.Client.call(Client.java:1412)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
at com.sun.proxy.$Proxy10.getFileInfo(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy11.getFileInfo(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2108)
at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1305)
at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1301)
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1317)
at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1426)
at org.apache.hadoop.examples.ProxyCreateSecureHDFSFile$1.run(ProxyCreateSecureHDFSFile.java:133)
at org.apache.hadoop.examples.ProxyCreateSecureHDFSFile$1.run(ProxyCreateSecureHDFSFile.java:123)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.examples.ProxyCreateSecureHDFSFile.main(ProxyCreateSecureHDFSFile.java:123)
Caused by: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]
at org.apache.hadoop.ipc.Client$Connection$1.run(Client.java:687)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.ipc.Client$Connection.handleSaslConnectionFailure(Client.java:650)
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:737)
at org.apache.hadoop.ipc.Client$Connection.access$2900(Client.java:375)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1528)
at org.apache.hadoop.ipc.Client.call(Client.java:1451)
... 23 more
Caused by: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]
at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:172)
at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:396)
at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:560)
at org.apache.hadoop.ipc.Client$Connection.access$1900(Client.java:375)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:729)
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:725)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:725)
... 26 more
This is my source code.
package org.apache.hadoop.examples;
import java.io.*;
import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
public class ProxyCreateSecureHDFSFile {
static String principal;
static String keytab;
public static void writeFileToHDFS(String in, String out, FileSystem fs) throws IOException {
//read local file line by line and write to hdfs
FileSystem hdfs =fs;
BufferedReader br = new BufferedReader(new FileReader(in));
String line = "";
StringBuilder aux = new StringBuilder();
while ((line = br.readLine()) != null) {
aux.append(line).append("\n");
}
byte[] byt1=aux.toString().getBytes();
FSDataOutputStream fsOutStream3 = hdfs.create(new Path(out));
fsOutStream3.write(byt1);
fsOutStream3.close();
br.close();
}
public static void main(String[] args) throws IOException {
if (args.length < 6){
System.out.println("Usage : <output-path> <principal> <keytab> <core-site.xml> <hdfs-site.xml> <mapred-site.xml>");
System.exit(1);
}
final String localInputPath = args[0];
final String outputArg = args[1];
final Path outputPath = new Path(outputArg);
final String principal = args[2];
final String keytab = args[3];
final Configuration conf = new Configuration();
try {
conf.set("hadoop.security.authentication", "Kerberos");
conf.set("debug", "true");
conf.set("sun.security.krb5.debug", "true");
conf.set("sun.security.spnego.debug", "true");
conf.set("hadoop.rpc.protection","authentication,privacy");
conf.addResource(new Path(args[4]));
conf.addResource(new Path(args[5]));
conf.addResource(new Path(args[6]));
UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
UserGroupInformation proxy =
UserGroupInformation.createProxyUser("user#XXXXX.XXXXX.COM",
UserGroupInformation.getLoginUser());
System.out.println(proxy.getUserName());
System.out.println(UserGroupInformation.getLoginUser().toString());
System.out.println(UserGroupInformation.getCurrentUser().toString());
proxy.doAs(new PrivilegedExceptionAction<Void>() {
public Void run() throws Exception {
FileSystem hdfs = FileSystem.get(conf);
Path homeDir=hdfs.getHomeDirectory();
System.out.println("Home Directory : " +homeDir);
Path workingDir=hdfs.getWorkingDirectory();
System.out.println("Working Directory : " +workingDir);
if(hdfs.exists(outputPath)){
hdfs.delete(outputPath, true);
}
writeFileToHDFS("/home/user/file.txt", outputArg+"/file.txt", hdfs);
return null;
}
});
} catch (Exception e) {
System.out.println("--------EXCEPTION________________");
e.printStackTrace();
}
}}
I have to make sure this is working as a standard java application because the client machines won't have hadoop libraries installed.

Related

Java agent in Lotus notes to connect to FileNet P8

i am trying to developp an app in lotus notes to connect to filenetp8, and manage the documents.
anyway i was using the filenet java api and did really connect from a distant machine but through a java app.
anad when it comes to lotus i'm stuck
this is the error output
log4j:WARN Caught Exception while in Loader.getResource. This may be innocuous.
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
at java.lang.reflect.Method.invoke(Method.java:611)
at org.apache.log4j.helpers.Loader.getTCL(Loader.java:154)
at org.apache.log4j.helpers.Loader.getResource(Loader.java:92)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:103)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at org.apache.log4j.Logger.getLogger(Logger.java:104)
at com.filenet.apiimpl.util.BaseLogger.getErrorLogger(BaseLogger.java:507)
at com.filenet.apiimpl.util.BaseLogger.info(BaseLogger.java:388)
at com.filenet.apiimpl.util.ConfigValueLookup.<clinit>(ConfigValueLookup.java:1268)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at com.filenet.apiimpl.util.BaseLogger.<clinit>(BaseLogger.java:46)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at com.filenet.api.util.UserContext.<clinit>(UserContext.java:88)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at JavaAgent.<init>(Unknown Source)
at java.lang.J9VMInternals.newInstanceImpl(Native Method)
at java.lang.Class.newInstance(Class.java:1547)
at lotus.domino.AgentInfo.newInstance(Unknown Source)
at lotus.domino.AgentLauncher.run(Unknown Source)
at lotus.domino.NotesThread.run(Unknown Source)
Caused by: java.security.AccessControlException: Access denied (java.lang.RuntimePermission getClassLoader)
at java.security.AccessController.throwACE(AccessController.java:100)
at java.security.AccessController.checkPermission(AccessController.java:174)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:544)
at COM.ibm.JEmpower.applet.AppletSecurity.superDotCheckPermission(AppletSecurity.java:1449)
at COM.ibm.JEmpower.applet.AppletSecurity.checkRuntimePermission(AppletSecurity.java:1311)
at COM.ibm.JEmpower.applet.AppletSecurity.checkPermission(AppletSecurity.java:1611)
at COM.ibm.JEmpower.applet.AppletSecurity.checkPermission(AppletSecurity.java:1464)
at java.lang.Thread.getContextClassLoader(Thread.java:458)
... 27 more
log4j:WARN Caught Exception while in Loader.getResource. This may be innocuous.
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
at java.lang.reflect.Method.invoke(Method.java:611)
at org.apache.log4j.helpers.Loader.getTCL(Loader.java:154)
at org.apache.log4j.helpers.Loader.getResource(Loader.java:92)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:105)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at org.apache.log4j.Logger.getLogger(Logger.java:104)
at com.filenet.apiimpl.util.BaseLogger.getErrorLogger(BaseLogger.java:507)
at com.filenet.apiimpl.util.BaseLogger.info(BaseLogger.java:388)
at com.filenet.apiimpl.util.ConfigValueLookup.<clinit> (ConfigValueLookup.java:1268)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at com.filenet.apiimpl.util.BaseLogger.<clinit>(BaseLogger.java:46)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at com.filenet.api.util.UserContext.<clinit>(UserContext.java:88)
at java.lang.J9VMInternals.initializeImpl(Native Method)
at java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
at JavaAgent.<init>(Unknown Source)
at java.lang.J9VMInternals.newInstanceImpl(Native Method)
at java.lang.Class.newInstance(Class.java:1547)
at lotus.domino.AgentInfo.newInstance(Unknown Source)
at lotus.domino.AgentLauncher.run(Unknown Source)
at lotus.domino.NotesThread.run(Unknown Source)
Caused by: java.security.AccessControlException: Access denied (java.lang.RuntimePermission getClassLoader)
at java.security.AccessController.throwACE(AccessController.java:100)
at java.security.AccessController.checkPermission(AccessController.java:174)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:544)
at COM.ibm.JEmpower.applet.AppletSecurity.superDotCheckPermission(AppletSecurity.java:1449)
at COM.ibm.JEmpower.applet.AppletSecurity.checkRuntimePermission(AppletSecurity.java:1311)
at COM.ibm.JEmpower.applet.AppletSecurity.checkPermission(AppletSecurity.java:1611)
at COM.ibm.JEmpower.applet.AppletSecurity.checkPermission(AppletSecurity.java:1464)
at java.lang.Thread.getContextClassLoader(Thread.java:458)
... 27 more
log4j:WARN No appenders could be found for logger (filenet_error.api.com.filenet.apiimpl.util.ConfigValueLookup).
log4j:WARN Please initialize the log4j system properly.
http://172.17.20.155:9080/wsi/FNCEWS40MTOM
FileNetP8WSI
gci_admin
Passw0rd
Invalid login credentials supplied - please try again
com.filenet.api.exception.EngineRuntimeException: FNRCE0040E: E_NOT_AUTHENTICATED: The user is not authenticated. Message was: Access denied (javax.security.auth.AuthPermission createLoginContext.FileNetP8WSI)
at com.filenet.apiimpl.core.UserPasswordToken.getSubject(UserPasswordToken.java:127)
at com.filenet.api.util.UserContext.createSubject(UserContext.java:288)
at JavaAgent.establishConnection(Unknown Source)
at JavaAgent.NotesMain(Unknown Source)
at lotus.domino.AgentBase.runNotes(Unknown Source)
at lotus.domino.NotesThread.run(Unknown Source)
Caused by: java.security.AccessControlException: Access denied (javax.security.auth.AuthPermission createLoginContext.FileNetP8WSI)
at java.security.AccessController.throwACE(AccessController.java:100)
at java.security.AccessController.checkPermission(AccessController.java:174)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:544)
at COM.ibm.JEmpower.applet.AppletSecurity.superDotCheckPermission(AppletSecurity.java:1449)
at COM.ibm.JEmpower.applet.AppletSecurity.checkPermission(AppletSecurity.java:1617)
at COM.ibm.JEmpower.applet.AppletSecurity.checkPermission(AppletSecurity.java:1464)
at javax.security.auth.login.LoginContext.init(LoginContext.java:247)
at javax.security.auth.login.LoginContext.<init>(LoginContext.java:426)
at com.filenet.apiimpl.core.UserPasswordToken$1.run(UserPasswordToken.java:147)
at java.security.AccessController.doPrivileged(AccessController.java:327)
at com.filenet.apiimpl.core.UserPasswordToken.getPrivilegedLoginContext(UserPasswordToken.java:152)
at com.filenet.apiimpl.core.UserPasswordToken.getSubject(UserPasswordToken.java:112)
... 5 more
and this is my agent code
import java.util.Iterator;
import java.util.Vector;
import javax.security.auth.Subject;
import com.filenet.api.collection.ObjectStoreSet;
import com.filenet.api.core.Connection;
import com.filenet.api.core.Domain;
import com.filenet.api.core.Factory;
import com.filenet.api.core.ObjectStore;
import com.filenet.api.exception.EngineRuntimeException;
import com.filenet.api.exception.ExceptionCode;
import com.filenet.api.util.UserContext;
import lotus.domino.*;
public class JavaAgent extends AgentBase {
public Connection con = null;
public UserContext uc = UserContext.get();
Domain dom = null;
String domainName = null;
ObjectStoreSet ost = null;
Vector<String> osnames = new Vector<String>();
boolean isConnected = false;
public void establishConnection(String userName, String password,
String stanza, String uri) {
System.out.println("entree");
con = Factory.Connection.getConnection(uri);
Subject sub = UserContext
.createSubject(con, userName, password, stanza);
System.out.println("sortie");
uc.pushSubject(sub);
dom = fetchDomain();
domainName = dom.get_Name();
ost = getOSSet();
}
/*
* Returns Domain object.
*/
public Domain fetchDomain() {
dom = Factory.Domain.fetchInstance(con, null, null);
return dom;
}
/*
* Returns ObjectStoreSet from Domain
*/
public ObjectStoreSet getOSSet() {
ost = dom.get_ObjectStores();
return ost;
}
/*
* Returns vector containing ObjectStore names from object stores available
* in ObjectStoreSet.
*/
public Vector<String> getOSNames() {
if (osnames.isEmpty()) {
Iterator<?> it = ost.iterator();
while (it.hasNext()) {
ObjectStore os = (ObjectStore) it.next();
osnames.add(os.get_DisplayName());
}
}
return osnames;
}
/*
* Checks whether connection has established with the Content Engine or not.
*/
public boolean isConnected() {
return isConnected;
}
/*
* Returns ObjectStore object for supplied object store name.
*/
public ObjectStore fetchOS(String name) {
ObjectStore os = Factory.ObjectStore.fetchInstance(dom, name, null);
return os;
}
/*
* Returns the domain name.
*/
public String getDomainName() {
return domainName;
}
public void NotesMain() {
try {
Session session = getSession();
AgentContext agentContext = session.getAgentContext();
String uri = "http://172.17.20.155:9080/wsi/FNCEWS40MTOM";
String stanza = "FileNetP8WSI";
String user = "gci_admin";
String password = "Passw0rd";
System.out.println(uri);
System.out.println(stanza);
System.out.println(user);
System.out.println(password);
// (Your code goes here)
try {
establishConnection(user, password, stanza, uri);
System.out.println(getDomainName());
System.out.println(getOSNames());
} catch (EngineRuntimeException e) {
if (e.getExceptionCode() == ExceptionCode.E_NOT_AUTHENTICATED) {
System.out
.println("Invalid login credentials supplied - please try again");
} else if (e.getExceptionCode() == ExceptionCode.API_UNABLE_TO_USE_CONNECTION) {
System.out
.println("Unable to connect to server. Please check to see that URL is correct and server is running");
} else {
System.out.println(e.getMessage());
}
e.printStackTrace();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
Look at http://www-304.ibm.com/support/knowledgecenter/SSNW2F_5.2.0/com.ibm.p8.ce.dev.ce.doc/gs_concepts_setup_thick_client.htm
You need to look at CEWS transport protocol chapter.
Check the JAAS configuration file
Check the JVM JAAS params.
Or you can try to use IIOP transport ... or you can develop your own JAAS login module to exclude login/password from source code.
I know this is an old question, but i thought because i have ran into this same issue soo many times i have to post the answer somewhere, so here it is.
ADD
-Djava.security.auth.login.config=C:\..\..\FileNet\BPMClient\jaas.conf.WebSphere
to the JVM arguments of the server
==================================================
STEPS TO ADDING THE JVM ARG THROUGH ADMIN CONSOLE
- In the Administration Console select Servers
- Expand Server Type and select WebSphere application servers
- Click on the name of your server
- Expand Java and Process Management and select Process Definition.
- Under the Additional Properties section, click Java Virtual Machine.
- Scroll down and locate the textbox for Generic JVM arguments.

Apache Spark eventLog configuration on Windows Giving error

I am using Spark 1.5 on Windows. I haven't installed any separate binaries of Hadoop.
I running a Master and a single worker.
It's a simple HelloWorld Program as below :
package com.java.spark;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
public class HelloWorld implements Serializable{
/**
*
*/
private static final long serialVersionUID = -7926281781224763077L;
public static void main(String[] args) {
// Local mode
//SparkConf sparkConf = new SparkConf().setAppName("HelloWorld").setMaster("local");
SparkConf sparkConf = new SparkConf().setAppName("HelloWorld").setMaster("spark://192.168.1.106:7077")
.set("spark.eventLog.enabled", "true")
.set("spark.eventLog.dir", "file:///D:/SparkEventLogsHistory");
//.set("spark.eventLog.dir", "/work/");
//tried many combinations above but all gives error.
JavaSparkContext ctx = new JavaSparkContext(sparkConf);
String[] arr = new String[] { "John", "Paul", "Gavin", "Rahul", "Angel" };
List<String> inputList = Arrays.asList(arr);
JavaRDD<String> inputRDD = ctx.parallelize(inputList);
inputRDD.foreach(new VoidFunction<String>() {
public void call(String input) throws Exception {
System.out.println(input);
}
});
}
}
The exception I am getting is :
Exception in thread "main" java.io.IOException: Cannot run program "cygpath": CreateProcess error=2, The system cannot find the file specified
at java.lang.ProcessBuilder.start(Unknown Source)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:206)
at org.apache.hadoop.util.Shell.run(Shell.java:188)
at org.apache.hadoop.fs.FileUtil$CygPathCommand.<init>(FileUtil.java:412)
at org.apache.hadoop.fs.FileUtil.makeShellPath(FileUtil.java:438)
at org.apache.hadoop.fs.FileUtil.makeShellPath(FileUtil.java:465)
at org.apache.hadoop.fs.RawLocalFileSystem.execCommand(RawLocalFileSystem.java:592)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:584)
at org.apache.hadoop.fs.FilterFileSystem.setPermission(FilterFileSystem.java:420)
at org.apache.spark.scheduler.EventLoggingListener.start(EventLoggingListener.scala:130)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:541)
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:61)
at com.java.spark.HelloWorld.main(HelloWorld.java:28)
Caused by: java.io.IOException: CreateProcess error=2, The system cannot find the file specified
at java.lang.ProcessImpl.create(Native Method)
at java.lang.ProcessImpl.<init>(Unknown Source)
at java.lang.ProcessImpl.start(Unknown Source)
... 13 more
16/04/01 20:13:24 INFO ShutdownHookManager: Shutdown hook called
Does anyone has any idea how to resolve this exception, so that Spark can pick the eventLogs from local directory.
If I dont give configure eventLog.dir then exception changes to :
Exception in thread "main" java.io.FileNotFoundException: File file:/H:/tmp/spark-events does not exist
at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:468)
at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:373)
at org.apache.spark.scheduler.EventLoggingListener.start(EventLoggingListener.scala:100)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:541)
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:61)
at com.java.spark.HelloWorld.main(HelloWorld.java:28)

Java with HDFS file read/write

I am new to Hadoop and Java. I have to read and write to a *.txt file stored on HDFS in my remote cloud-era distribution. And for the same I have this small java program written:
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class ReadHadoopFileData {
public static void main(String[] args) throws IOException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem hdfs = FileSystem.get( new URI( "hdfs://admin:H4d00p#172.16.10.124:8888" ), configuration );
Path file = new Path("hdfs://admin:H4d00p#172.16.10.124:8888/user/admin/Data/Tlog.txt");
try{
BufferedReader br=new BufferedReader(new InputStreamReader(hdfs.open(file)));
String line;
line=br.readLine();
while (line != null){
System.out.println(line);
line=br.readLine();
}
}catch(Exception e){
e.printStackTrace();
}
}
}
But when the row BufferedReader br=new BufferedReader(new InputStreamReader(hdfs.open(file))); is executed I am running into this error:
java.io.IOException: Failed on local exception: com.google.protobuf.InvalidProtocolBufferException: Protocol message tag had invalid wire type.; Host Details : local host is: "KWTLT02221/169.254.208.16"; destination host is: "172.16.104.124":8888;
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:772)
at org.apache.hadoop.ipc.Client.call(Client.java:1472)
at org.apache.hadoop.ipc.Client.call(Client.java:1399)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232)
at com.sun.proxy.$Proxy9.getBlockLocations(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:254)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy10.getBlockLocations(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1220)
at org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1210)
at org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1200)
at org.apache.hadoop.hdfs.DFSInputStream.fetchLocatedBlocksAndGetLastBlockLength(DFSInputStream.java:271)
at org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:238)
at org.apache.hadoop.hdfs.DFSInputStream.<init>(DFSInputStream.java:231)
at org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1498)
at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:302)
at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:298)
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:298)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:766)
at ReadHadoopFileData.main(ReadHadoopFileData.java:26)
Caused by: com.google.protobuf.InvalidProtocolBufferException: Protocol message tag had invalid wire type.
at com.google.protobuf.InvalidProtocolBufferException.invalidWireType(InvalidProtocolBufferException.java:99)
at com.google.protobuf.UnknownFieldSet$Builder.mergeFieldFrom(UnknownFieldSet.java:498)
at com.google.protobuf.UnknownFieldSet$Builder.mergeFrom(UnknownFieldSet.java:461)
at com.google.protobuf.UnknownFieldSet$Builder.mergeFrom(UnknownFieldSet.java:579)
at com.google.protobuf.UnknownFieldSet$Builder.mergeFrom(UnknownFieldSet.java:280)
at com.google.protobuf.CodedInputStream.readGroup(CodedInputStream.java:240)
at com.google.protobuf.UnknownFieldSet$Builder.mergeFieldFrom(UnknownFieldSet.java:488)
at com.google.protobuf.GeneratedMessage.parseUnknownField(GeneratedMessage.java:193)
at org.apache.hadoop.ipc.protobuf.RpcHeaderProtos$RpcResponseHeaderProto.<init>(RpcHeaderProtos.java:2207)
at org.apache.hadoop.ipc.protobuf.RpcHeaderProtos$RpcResponseHeaderProto.<init>(RpcHeaderProtos.java:2165)
at org.apache.hadoop.ipc.protobuf.RpcHeaderProtos$RpcResponseHeaderProto$1.parsePartialFrom(RpcHeaderProtos.java:2295)
at org.apache.hadoop.ipc.protobuf.RpcHeaderProtos$RpcResponseHeaderProto$1.parsePartialFrom(RpcHeaderProtos.java:2290)
at com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:200)
at com.google.protobuf.AbstractParser.parsePartialDelimitedFrom(AbstractParser.java:241)
at com.google.protobuf.AbstractParser.parseDelimitedFrom(AbstractParser.java:253)
at com.google.protobuf.AbstractParser.parseDelimitedFrom(AbstractParser.java:259)
at com.google.protobuf.AbstractParser.parseDelimitedFrom(AbstractParser.java:49)
at org.apache.hadoop.ipc.protobuf.RpcHeaderProtos$RpcResponseHeaderProto.parseDelimitedFrom(RpcHeaderProtos.java:3167)
at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1072)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:966)
Could someone help me out to get this resolved please ? I am on this for a day now.
I figured out the solution for this error. And looks like I was using the wrong port. I was using the port number as I see on HUE URL ( misleaded from different sources).
If I chose the port number as defined for the configuration "NameNode Service RPC Port" OR "dfs.namenode.servicerpc-address" on the name node from Cloudera manager, it works fine.

Null Pointer Exception - Eclipse remote Hadoop

I have been seaching for days now, Please HELP
I have a nullpointerexception when executing a simple Wordcount for Hadoop (I have installed a cluster of VMs Ubuntu 14 of hadoop and run eclipse LOCALY)
java.lang.NullPointerException
at java.lang.ProcessBuilder.start(Unknown Source)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:482)
at org.apache.hadoop.util.Shell.run(Shell.java:455)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:715)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:808)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:791)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:656)
at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:444)
at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:293)
at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:133)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:437)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1296)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1293)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Unknown Source)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1293)
at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:562)
at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:557)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Unknown Source)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:557)
at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:548)
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:833)
at WordCount.main(WordCount.java:76)
This is my code :
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
//import org.apache.log4j.Logger;
public class WordCount {
public static void main(String[] args) throws IOException {
JobConf conf = new JobConf(WordCount.class);
System.setProperty("HADOOP_USER_NAME", "hduser");
// specify output types
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
conf.set("hadoop.job.ugi", "hduser");
conf.set("fs.defaultFS", "hdfs://10.99.99.2:54310/user/hduser");
//log4j.logger.org.apache.hadoop = DEBUG
conf.set("mapred.job.tracker", "10.99.99.2:54311");
// specify input and output dirs
FileInputFormat.addInputPath(conf, new Path("input")); //hdfs://10.99.99.2:54310/user/hduser/input/Good.txt
FileOutputFormat.setOutputPath(conf, new Path("output"));
// specify a mapper
conf.setMapperClass(WordCountMapper.class);
// specify a reducer
conf.setReducerClass(WordCountReducer.class);
conf.setCombinerClass(WordCountReducer.class);
FileSystem fs = FileSystem.get(conf);
// CREATE FILE AND PRINT PATH TO CHECK IS EVERYTHING IS OK
fs.createNewFile(new Path("/user/hduser/test"));
FileStatus[] status = fs.listStatus(new Path("/user/hduser"));
for(int i=0;i<status.length;i++){
System.out.println(status[i].getPath());
}
// READ TXT TO BE SURE THERE IS NOT PERMISSION PROBLEM
Path pt= new Path("hdfs://10.99.99.2:54310/user/hduser/input/Good.txt");
BufferedReader br=new BufferedReader(new InputStreamReader(fs.open(pt)));
String line;
line=br.readLine();
while (line != null){
System.out.println(line);
line=br.readLine(); }
JobClient client = new JobClient(conf);
client.setConf(conf);
try {
JobClient.runJob(conf);
} catch (Exception e) {
e.printStackTrace();
}
}
}
The code looks fine! Please make sure you are giving an accurate command to reply the JAR file and also you can make some tests and debug them to understand where the null pointer Exception is coming from.

- RMI code base doesn't work - Registry Lookup very slow and doesn't work

I can not understand how to use the codebase.
I have the following structure:
CLIENT:
Client.java
client.policy
SERVER:
Server.java
Hello.java
HelloInterfaces.java
HelloInterfaces.java
public interface HelloInterfaces extends Remote {
}
Hello.java
public class Hello extends UnicastRemoteObject implements HelloInterfaces{
public Hello() throws RemoteException{
}
}
Server.java
public class Server {
public static void main(String[] arg) throws RemoteException {
Registry reg = LocateRegistry.createRegistry(1099);
reg.rebind("Hello", new Hello());
System.out.println("Ready");
}
}
Client.java
public class Client {
public static void main(String[] args) throws RemoteException, NotBoundException, MalformedURLException {
String host="192.168.1.227"; //IP SERVER
System.setProperty("java.rmi.server.codebase", "http://" + host + ":8080");
System.setProperty("java.security.policy", "client.policy");
System.setSecurityManager(new RMISecurityManager());
Registry reg = LocateRegistry.getRegistry(host);
System.out.println("Regitry Found!");
System.out.println(reg.list().length);
System.out.println(reg.list()[0]);
HelloInterfaces h;
h = (HelloInterfaces) reg.lookup("Hello");
System.out.println(h);
}
}
client.policy
grant {
permission java.security.AllPermission;
};
Execute on server (Virtual Machine with Linux) directly from Netbeans:
Output Server: Ready!
On the client, I have compiled the source with option --classpath .:Server.jar (where Server.jar contains HelloInterfaces.class).
execute Client:
Output Client:
Registry Found!
1
Hello
(Running......)
and after two minutes:
Exception in thread "main" java.rmi.UnmarshalException: error unmarshalling return; nested exception is:
java.lang.ClassNotFoundException: HelloInterfaces
at sun.rmi.registry.RegistryImpl_Stub.lookup(Unknown Source)
at Client.main(Client.java:36)
Caused by: java.lang.ClassNotFoundException: HelloInterfaces
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.rmi.server.LoaderHandler$Loader.loadClass(LoaderHandler.java:1206)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:270)
at sun.rmi.server.LoaderHandler.loadClassForName(LoaderHandler.java:1219)
at sun.rmi.server.LoaderHandler.loadProxyInterfaces(LoaderHandler.java:729)
at sun.rmi.server.LoaderHandler.loadProxyClass(LoaderHandler.java:673)
at sun.rmi.server.LoaderHandler.loadProxyClass(LoaderHandler.java:610)
at java.rmi.server.RMIClassLoader$2.loadProxyClass(RMIClassLoader.java:646)
at java.rmi.server.RMIClassLoader.loadProxyClass(RMIClassLoader.java:311)
at sun.rmi.server.MarshalInputStream.resolveProxyClass(MarshalInputStream.java:255)
at java.io.ObjectInputStream.readProxyDesc(ObjectInputStream.java:1558)
at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1514)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1771)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370)
... 2 more
Also, on directory server that contains .class, I run a python httpserver on port 8080.
(same result with a NanoHTTP, http server written in java)
The app client contacts the http server because python httpserver output:
192.168.1.24 - - "GET / HTTP/1.1" 200 -
A codebase is a list of URLs of JAR files or directories that contain package-structured hierarchies of .class files. It doesn't seem likely that a host-and-port-only URL meets that description.

Resources