03讀取文件並排序

#!/usr/bin/python
# -*- coding:UTF-8 -*-
# 文件名：ReadFile.py
__author__ = "haohao"

# 讀取文件並排序

from pyspark import SparkConf
from pyspark import SparkContext

conf = SparkConf().setAppName("hao").setMaster("local[*]")
sc = SparkContext(conf=conf)

rdd = sc.textFile("../data/english.txt", minPartitions=4)       # 並行度
result = rdd.flatMap(lambda line: line.split()).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a+b)
# print(result)

# # 大數據中不使用以下這種方法，內存可能會被撐爆
# result = result.collect()
# sort_result = sorted(result, key=lambda s: s[1], reverse=True)
# print(sort_result)

# # 這個還是在spark集羣裏面跑，跑完再用collect把結果拉過來
# sort = result.sortBy(lambda e: e[1], ascending=False).collect()
# sort = result.sortByKey().collect()
sort = result.map(lambda pairs: (pairs[1], pairs[0])).sortByKey(ascending=False).map(lambda pairs: (pairs[1], pairs[0]))
sort.saveAsTextFile("./result")