1. 併發編程
- 在 Java 中，多線程訪問共享數據的時候會存在【線程安全】問題
- Scala 的多線程使用了新的通信機制
- 通過發送消息來通信，沒有了共享數據，從而實現併發編程
- Scala 使用的是 Akka 框架，Akka 通過 Actor 模式實現高併發
- Akka 是使用 Scala 語言編寫的用法高併發的編程框架
- Akka 的高併發是由 Actor 與 Actor 之間的通信
- Akka 模型
- 消息傳遞
- 併發(FIFO)
- 容錯
- Actor 是由 ActorSystem 來創建的
- ActorSystem 是單例的
- Actor 是多例的
- 一個 JVM 中，只需要有一個 ActorSystem
類型別名
[Scala] 純文本查看 複製代碼
|
1
2
3
4
5
6
7
|
object Alias { def main(args: Array[String]): Unit = { val arr1: Array[Int] = Array(1, 2, 3, 4, 5) type myArr = Array[Int] val arr2: myArr = Array(1, 2, 3, 4, 5) }} |
Akka 案例
- 創建 maven 工程，導入依賴
[XML] 純文本查看 複製代碼
|
001
002
003
004
005
006
007
008
009
010
011
012
013
014
015
016
017
018
019
020
021
022
023
024
025
026
027
028
029
030
031
032
033
034
035
036
037
038
039
040
041
042
043
044
045
046
047
048
049
050
051
052
053
054
055
056
057
058
059
060
061
062
063
064
065
066
067
068
069
070
071
072
073
074
075
076
077
078
079
080
081
082
083
084
085
086
087
088
089
090
091
092
093
094
095
096
097
098
099
100
101
102
103
104
105
|
<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.study</groupId> <artifactId>ScalaDemo</artifactId> <version>1.0-SNAPSHOT</version> <!-- 定義一下常量 --> <properties> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> <encoding>UTF-8</encoding> <scala.version>2.11.8</scala.version> <scala.compat.version>2.11</scala.compat.version> <akka.version>2.4.17</akka.version> </properties> <dependencies> <!-- 添加scala的依賴 --> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> <version>${scala.version}</version> </dependency> <!-- 添加akka的actor依賴 --> <dependency> <groupId>com.typesafe.akka</groupId> <artifactId>akka-actor_${scala.compat.version}</artifactId> <version>${akka.version}</version> </dependency> <!-- 多進程之間的Actor通信 --> <dependency> <groupId>com.typesafe.akka</groupId> <artifactId>akka-remote_${scala.compat.version}</artifactId> <version>${akka.version}</version> </dependency> </dependencies> <!-- 指定插件--> <build> <plugins> <!-- 指定編譯scala的插件 --> <plugin> <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> <version>3.2.2</version> <executions> <execution> <goals> <goal>compile</goal> <goal>testCompile</goal> </goals> <configuration> <args> <arg>-dependencyfile</arg> <arg>${project.build.directory}/.scala_dependencies</arg> </args> </configuration> </execution> </executions> </plugin> <!-- maven打包的插件 --> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <version>2.4.3</version> <executions> <execution> <phase>package</phase> <goals> <goal>shade</goal> </goals> <configuration> <filters> <filter> <artifact>*:*</artifact> <excludes> <exclude>META-INF/*.SF</exclude> <exclude>META-INF/*.DSA</exclude> <exclude>META-INF/*.RSA</exclude> </excludes> </filter> </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> <resource>reference.conf</resource> </transformer> <!-- 指定maven方法 --> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass></mainClass> </transformer> </transformers> </configuration> </execution> </executions> </plugin> </plugins> </build></project> |
- HelloActor入門案例
[Scala] 純文本查看 複製代碼
|
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
|
import akka.actor.{Actor, ActorRef, ActorSystem, Props}import com.typesafe.config.{Config, ConfigFactory}class HelloAcrot extends Actor { //type Receive = scala.PartialFunction[scala.Any, scala.Unit] override def receive: Receive = { case msg: String => println(msg) }}object HelloAcrot { val actorSystemName = "helloActorSystem" val actorName = "helloActor" def main(args: Array[String]): Unit = { val host = "127.0.0.1" val port = 8888 // 指定配置 host + port val prot = s""" |akka.actor.provider = "akka.remote.RemoteActorRefProvider" |akka.remote.netty.tcp.hostname = ${host} |akka.remote.netty.tcp.port = ${port} """ .stripMargin // 獲取一個配置實例 val config: Config = ConfigFactory.parseString(prot) // 創建一個ActorSystem val actorSystem: ActorSystem = ActorSystem.create(actorSystemName,config) val actor: ActorRef = actorSystem.actorOf(Props(new HelloAcrot()),actorName) actor ! "Hello Dog!" }} |
- Actor 互相通信
[Scala] 純文本查看 複製代碼
|
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
|
import akka.actor.{Actor, ActorRef, ActorSystem, Props}import com.typesafe.config.{Config, ConfigFactory}class Ai extends Actor { override def receive: Receive = { case str: String => { println(s"Ai獲取到消息${str}") sender() ! "Ai回覆你" } }}object Ai { val actorSystemName = "Ai_system" val actorName = "Ai_actor" def main(args: Array[String]): Unit = { val host = "127.0.0.1" val port = 9999 // 指定協議 val str = s""" |akka.actor.provider = "akka.remote.RemoteActorRefProvider" |akka.remote.netty.tcp.hostname = ${host} |akka.remote.netty.tcp.port = ${port} """ .stripMargin val config: Config = ConfigFactory.parseString(str) val actorSystem: ActorSystem = ActorSystem.create(actorSystemName, config) actorSystem.actorOf(Props(new Ai()), actorName) }} |
[Scala] 純文本查看 複製代碼
|
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
|
import akka.actor.{Actor, ActorSelection, ActorSystem, Props}import com.typesafe.config.{Config, ConfigFactory}class Horse extends Actor { override def receive: Receive = { case str:String => { println(s"獲取到Ai的回覆:${str}") sender() ! "Ai，你好" } } override def preStart(): Unit = { val path = s"akka.tcp://${Ai.actorSystemName}@127.0.0.1:9999/user/${Ai.actorName}" val actorProxy: ActorSelection = context.actorSelection(path) println(s"獲取${Horse.actorName}代理對象成功!") actorProxy ! "Ai你好" }}object Horse { val actorSystemName = "Horse_system" val actorName = "Horse_actor" def main(args: Array[String]): Unit = { val host = "127.0.0.1" val port = 8888 // 指定協議 val str = s""" |akka.actor.provider = "akka.remote.RemoteActorRefProvider" |akka.remote.netty.tcp.hostname = ${host} |akka.remote.netty.tcp.port = ${port} """ .stripMargin // 獲取 ActorSystem 配置 val config: Config = ConfigFactory.parseString(str) // 創建一個 ActorSystem 實例 val actorSystem: ActorSystem = ActorSystem.create(actorSystemName, config) actorSystem.actorOf(Props(new Horse()), actorName) }} |
Spark 通信機制
- HDFS: NameNode + DataNode
- Spark: Master + Workder
Spark RPC 通信的基本機制
- Master 先啓動，然後 Worker 向 Master 建立連接
- Worker 向 Master 註冊
- 資源（cores,memory）
- workerId
- 用 case class 封裝相關的信息
- Master 接收 Worker 的消息後，保存 workerId
- 集合 Map[workerId,Info類]
- Master 返回註冊成功消息
- case object
- workerId接收到註冊成功消息後，定時發送心跳信息
- 定時任務 case class(workerId)
- Master 定時任務，在 Master 啓動之後立即啓動
- 清楚超時沒發心跳的 Worder
[Scala] 純文本查看 複製代碼
|
01
02
03
04
05
06
07
08
09
10
11
12
13
14
|
object RpcMessage {}"worker向master註冊的樣例類"case class Worker2Master(workerId: String)"master 返回給 worker 註冊成功消息"case object Registered"worker定時向master發送的心跳信息"case class HeartBeat(workerId: String)"master 啓動定時，定時檢測超時的master"case object CheckWorkerStataus |
[Scala] 純文本查看 複製代碼
|
01
02
03
04
05
06
07
08
09
10
11
12
13
14
|
object RpcMessage {}"worker向master註冊的樣例類"case class Worker2Master(workerId: String)"master 返回給 worker 註冊成功消息"case object Registered"worker定時向master發送的心跳信息"case class HeartBeat(workerId: String)"master 啓動定時，定時檢測超時的master"case object CheckWorkerStataus |
[Scala] 純文本查看 複製代碼
|
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
|
import java.util.UUIDimport akka.actor.{Actor, ActorSelection, ActorSystem, Props}import com.typesafe.config.{Config, ConfigFactory}class Worker(val masterHost: String, val masterPort: String) extends Actor { "workerId" val workerId: String = UUID.randomUUID().toString.replaceAll("-", "") override def receive: Receive = { case Registered => { "worker 接收到 master 返回的註冊成功消息" println(s"worder(${workerId})獲取到master的響應，註冊成功！") "worker 定時向 master 發送心跳" import scala.concurrent.duration._ "導入時間單位" import context.dispatcher context.system.scheduler.schedule(0 second, 10 second, sender(), HeartBeat(workerId)) } } override def preStart(): Unit = { val path = s"akka.tcp://${Master.actorSystemName}@${masterHost}:${masterPort}/user/${Master.actorName}" val masterProxy: ActorSelection = context.actorSelection(path) println(s"worker(${workerId})創建，成功獲取 master 代理") masterProxy ! Worker2Master(workerId) }}object Worker { val actorSystemName = "Worker_System" val actorName = "Worker_Actor" def main(args: Array[String]): Unit = { "參數校驗" if (args.length != 4) { println("Usage:com.study.sparkrpc.Worker<master host><master port><host><port>") sys.exit(1) } val Array(masterHost, masterPort, host, port) = args val str = s""" |akka.actor.provider = "akka.remote.RemoteActorRefProvider" |akka.remote.netty.tcp.hostname = ${host} |akka.remote.netty.tcp.port = ${port} """ .stripMargin val config: Config = ConfigFactory.parseString(str) val actorSystem: ActorSystem = ActorSystem.create(actorSystemName, config) actorSystem.actorOf(Props(new Worker(masterHost, masterPort)), actorName) }} |
[Scala] 純文本查看 複製代碼
|
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
import akka.actor.{Actor, ActorSystem, Props}import com.typesafe.config.{Config, ConfigFactory}import scala.collection.mutableclass Master extends Actor { "存儲所有的workerInfo" val workerMap = new mutable.HashMap[String, WorkerInfo]() override def preStart(): Unit = { import scala.concurrent.duration._ import context.dispatcher context.system.scheduler.schedule(0 second, 13 second, self, CheckWorkerStataus) } override def receive: Receive = { case Worker2Master(workerId) => { println(s"master 接收到 worker(${workerId}) 的註冊") val info = new WorkerInfo(workerId) info.lastHeartBeatTime = System.currentTimeMillis() workerMap.put(workerId, info) sender() ! Registered } case HeartBeat(workerId) => { workerMap(workerId).lastHeartBeatTime = System.currentTimeMillis() println(s"master 接收到 worker(${workerId})的心跳") |