import org.apache.spark.SparkConf
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
/**
* @author clebeg
*/
object FirstTry {
def main(args: Array[String]): Unit = {
val conf = new SparkConf
conf.setMaster("spark://yourip:7077")
conf.set("spark.app.name", "first-tryspark")
val sc = new SparkContext(conf)
val rawblocks = sc.textFile("hdfs://yourip:9000/user/hadoop/linkage")
println(rawblocks.first)
}
}
5 部分错误汇总
大部分问题上面已经提到，这里不多说，下面提几个主要的问题：
Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources 分析问题：点开运行ID对应的运行日志发现下面的错误：
15/10/10 08:49:01 INFO executor.CoarseGrainedExecutorBackend: Registered signal handlers for [TERM, HUP, INT]
15/10/10 08:49:01 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
15/10/10 08:49:02 INFO spark.SecurityManager: Changing view acls to: hadoop,Administrator
15/10/10 08:49:02 INFO spark.SecurityManager: Changing modify acls to: hadoop,Administrator
15/10/10 08:49:02 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(hadoop, Administrator); users with modify permissions: Set(hadoop, Administrator)
15/10/10 08:49:02 INFO slf4j.Slf4jLogger: Slf4jLogger started
15/10/10 08:49:02 INFO Remoting: Starting remoting
15/10/10 08:49:02 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://driverPropsFetcher@10.16.112.121:58708]
15/10/10 08:49:02 INFO util.Utils: Successfully started service 'driverPropsFetcher' on port 58708.
Exception in thread "main" java.lang.reflect.UndeclaredThrowableException
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1643)
at org.apache.spark.deploy.SparkHadoopUtil.runAsSparkUser(SparkHadoopUtil.scala:65)
at org.apache.spark.executor.CoarseGrainedExecutorBackend$.run(CoarseGrainedExecutorBackend.scala:146)
at org.apache.spark.executor.CoarseGrainedExecutorBackend$.main(CoarseGrainedExecutorBackend.scala:245)
at org.apache.spark.executor.CoarseGrainedExecutorBackend.main(CoarseGrainedExecutorBackend.scala)
Caused by: java.util.concurrent.TimeoutException: Futures timed out after [120 seconds]
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219)
at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223)
at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107)
at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
at scala.concurrent.Await$.result(package.scala:107)
at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:97)
at org.apache.spark.executor.CoarseGrainedExecutorBackend$$anonfun$run$1.apply$mcV$sp(CoarseGrainedExecutorBackend.scala:159)
at org.apache.spark.deploy.SparkHadoopUtil$$anon$1.run(SparkHadoopUtil.scala:66)
at org.apache.spark.deploy.SparkHadoopUtil$$anon$1.run(SparkHadoopUtil.scala:65)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
... 4 more
15/10/10 08:51:02 INFO util.Utils: Shutdown hook called
重庆分公司
重庆分公司
