Skip to content

Commit 477868e

Browse files
author
pinard.liu
committed
upload association code
1 parent 7a68b03 commit 477868e

2 files changed

Lines changed: 258 additions & 1 deletion

File tree

‎classic-machine-learning/fp_tree_prefixspan.ipynb‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"Copyright (C) 2016 - 2019 Pinard Liu(liujianping-ok@163.com)\n",
8+
"\n",
9+
"https://www.cnblogs.com/pinard\n",
10+
"\n",
11+
"Permission given to modify the code as long as you keep this declaration at the top\n",
12+
"\n",
13+
"用Spark学习FP Tree算法和PrefixSpan算法 https://www.cnblogs.com/pinard/p/6340162.html"
14+
]
15+
},
16+
{
17+
"cell_type": "code",
18+
"execution_count": 1,
19+
"metadata": {},
20+
"outputs": [],
21+
"source": [
22+
"import os\n",
23+
"import sys\n",
24+
"\n",
25+
"#下面这些目录都是你自己机器的Spark安装目录和Java安装目录\n",
26+
"os.environ['SPARK_HOME'] = \"C:/Tools/spark-2.2.0-bin-hadoop2.6/\"\n",
27+
"os.environ['PYSPARK_PYTHON'] = \"C:/Users/tata/AppData/Local/Programs/Python/Python36/python.exe\"\n",
28+
"\n",
29+
"sys.path.append(\"C:/Tools/spark-2.2.0-bin-hadoop2.6/bin\")\n",
30+
"sys.path.append(\"C:/Tools/spark-2.2.0-bin-hadoop2.6/python\")\n",
31+
"sys.path.append(\"C:/Tools/spark-2.2.0-bin-hadoop2.6/python/pyspark\")\n",
32+
"sys.path.append(\"C:/Tools/spark-2.2.0-bin-hadoop2.6/python/lib\")\n",
33+
"sys.path.append(\"C:/Tools/spark-2.2.0-bin-hadoop2.6/python/lib/pyspark.zip\")\n",
34+
"sys.path.append(\"C:/Tools/spark-2.2.0-bin-hadoop2.6/python/lib/py4j-0.10.4-src.zip\")\n",
35+
"sys.path.append(\"C:/Program Files/Java/jdk1.8.0_171\")\n",
36+
"\n",
37+
"from pyspark import SparkContext\n",
38+
"from pyspark import SparkConf\n",
39+
"\n",
40+
"\n",
41+
"sc = SparkContext(\"local\",\"testing\")"
42+
]
43+
},
44+
{
45+
"cell_type": "code",
46+
"execution_count": 2,
47+
"metadata": {},
48+
"outputs": [
49+
{
50+
"name": "stdout",
51+
"output_type": "stream",
52+
"text": [
53+
"<SparkContext master=local appName=testing>\n"
54+
]
55+
}
56+
],
57+
"source": [
58+
"print (sc)"
59+
]
60+
},
61+
{
62+
"cell_type": "code",
63+
"execution_count": 3,
64+
"metadata": {
65+
"scrolled": false
66+
},
67+
"outputs": [],
68+
"source": [
69+
"from pyspark.mllib.fpm import FPGrowth\n",
70+
"data = [[\"A\", \"B\", \"C\", \"E\", \"F\",\"O\"], [\"A\", \"C\", \"G\"], [\"E\",\"I\"], [\"A\", \"C\",\"D\",\"E\",\"G\"], [\"A\", \"C\", \"E\",\"G\",\"L\"],\n",
71+
" [\"E\",\"J\"],[\"A\",\"B\",\"C\",\"E\",\"F\",\"P\"],[\"A\",\"C\",\"D\"],[\"A\",\"C\",\"E\",\"G\",\"M\"],[\"A\",\"C\",\"E\",\"G\",\"N\"]]\n",
72+
"rdd = sc.parallelize(data, 2)\n",
73+
"#支持度阈值为20%\n",
74+
"model = FPGrowth.train(rdd, 0.2, 2)"
75+
]
76+
},
77+
{
78+
"cell_type": "code",
79+
"execution_count": 4,
80+
"metadata": {},
81+
"outputs": [
82+
{
83+
"data": {
84+
"text/plain": [
85+
"[FreqItemset(items=['A'], freq=8),\n",
86+
" FreqItemset(items=['B'], freq=2),\n",
87+
" FreqItemset(items=['B', 'A'], freq=2),\n",
88+
" FreqItemset(items=['B', 'C'], freq=2),\n",
89+
" FreqItemset(items=['B', 'C', 'A'], freq=2),\n",
90+
" FreqItemset(items=['B', 'E'], freq=2),\n",
91+
" FreqItemset(items=['B', 'E', 'A'], freq=2),\n",
92+
" FreqItemset(items=['B', 'E', 'C'], freq=2),\n",
93+
" FreqItemset(items=['B', 'E', 'C', 'A'], freq=2),\n",
94+
" FreqItemset(items=['C'], freq=8),\n",
95+
" FreqItemset(items=['C', 'A'], freq=8),\n",
96+
" FreqItemset(items=['D'], freq=2),\n",
97+
" FreqItemset(items=['D', 'A'], freq=2),\n",
98+
" FreqItemset(items=['D', 'C'], freq=2),\n",
99+
" FreqItemset(items=['D', 'C', 'A'], freq=2),\n",
100+
" FreqItemset(items=['E'], freq=8),\n",
101+
" FreqItemset(items=['E', 'A'], freq=6),\n",
102+
" FreqItemset(items=['E', 'C'], freq=6),\n",
103+
" FreqItemset(items=['E', 'C', 'A'], freq=6),\n",
104+
" FreqItemset(items=['F'], freq=2),\n",
105+
" FreqItemset(items=['F', 'A'], freq=2),\n",
106+
" FreqItemset(items=['F', 'B'], freq=2),\n",
107+
" FreqItemset(items=['F', 'B', 'A'], freq=2),\n",
108+
" FreqItemset(items=['F', 'B', 'C'], freq=2),\n",
109+
" FreqItemset(items=['F', 'B', 'C', 'A'], freq=2),\n",
110+
" FreqItemset(items=['F', 'B', 'E'], freq=2),\n",
111+
" FreqItemset(items=['F', 'B', 'E', 'A'], freq=2),\n",
112+
" FreqItemset(items=['F', 'B', 'E', 'C'], freq=2),\n",
113+
" FreqItemset(items=['F', 'B', 'E', 'C', 'A'], freq=2),\n",
114+
" FreqItemset(items=['F', 'C'], freq=2),\n",
115+
" FreqItemset(items=['F', 'C', 'A'], freq=2),\n",
116+
" FreqItemset(items=['F', 'E'], freq=2),\n",
117+
" FreqItemset(items=['F', 'E', 'A'], freq=2),\n",
118+
" FreqItemset(items=['F', 'E', 'C'], freq=2),\n",
119+
" FreqItemset(items=['F', 'E', 'C', 'A'], freq=2),\n",
120+
" FreqItemset(items=['G'], freq=5),\n",
121+
" FreqItemset(items=['G', 'A'], freq=5),\n",
122+
" FreqItemset(items=['G', 'C'], freq=5),\n",
123+
" FreqItemset(items=['G', 'C', 'A'], freq=5),\n",
124+
" FreqItemset(items=['G', 'E'], freq=4),\n",
125+
" FreqItemset(items=['G', 'E', 'A'], freq=4),\n",
126+
" FreqItemset(items=['G', 'E', 'C'], freq=4),\n",
127+
" FreqItemset(items=['G', 'E', 'C', 'A'], freq=4)]"
128+
]
129+
},
130+
"execution_count": 4,
131+
"metadata": {},
132+
"output_type": "execute_result"
133+
}
134+
],
135+
"source": [
136+
"sorted(model.freqItemsets().collect())"
137+
]
138+
},
139+
{
140+
"cell_type": "code",
141+
"execution_count": 5,
142+
"metadata": {},
143+
"outputs": [],
144+
"source": [
145+
"from pyspark.mllib.fpm import PrefixSpan\n",
146+
"data = [\n",
147+
" [['a'],[\"a\", \"b\", \"c\"], [\"a\",\"c\"],[\"d\"],[\"c\", \"f\"]],\n",
148+
" [[\"a\",\"d\"], [\"c\"],[\"b\", \"c\"], [\"a\", \"e\"]],\n",
149+
" [[\"e\", \"f\"], [\"a\", \"b\"], [\"d\",\"f\"],[\"c\"],[\"b\"]],\n",
150+
" [[\"e\"], [\"g\"],[\"a\", \"f\"],[\"c\"],[\"b\"],[\"c\"]]\n",
151+
" ]\n",
152+
"rdd = sc.parallelize(data, 2)\n",
153+
"model = PrefixSpan.train(rdd, 0.5,4)"
154+
]
155+
},
156+
{
157+
"cell_type": "code",
158+
"execution_count": 6,
159+
"metadata": {},
160+
"outputs": [
161+
{
162+
"data": {
163+
"text/plain": [
164+
"[FreqSequence(sequence=[['a']], freq=4),\n",
165+
" FreqSequence(sequence=[['a'], ['a']], freq=2),\n",
166+
" FreqSequence(sequence=[['a'], ['b']], freq=4),\n",
167+
" FreqSequence(sequence=[['a'], ['b'], ['a']], freq=2),\n",
168+
" FreqSequence(sequence=[['a'], ['b'], ['c']], freq=2),\n",
169+
" FreqSequence(sequence=[['a'], ['b', 'c']], freq=2),\n",
170+
" FreqSequence(sequence=[['a'], ['b', 'c'], ['a']], freq=2),\n",
171+
" FreqSequence(sequence=[['a'], ['c']], freq=4),\n",
172+
" FreqSequence(sequence=[['a'], ['c'], ['a']], freq=2),\n",
173+
" FreqSequence(sequence=[['a'], ['c'], ['b']], freq=3),\n",
174+
" FreqSequence(sequence=[['a'], ['c'], ['c']], freq=3),\n",
175+
" FreqSequence(sequence=[['a'], ['d']], freq=2),\n",
176+
" FreqSequence(sequence=[['a'], ['d'], ['c']], freq=2),\n",
177+
" FreqSequence(sequence=[['a'], ['f']], freq=2),\n",
178+
" FreqSequence(sequence=[['b']], freq=4),\n",
179+
" FreqSequence(sequence=[['b'], ['a']], freq=2),\n",
180+
" FreqSequence(sequence=[['b'], ['c']], freq=3),\n",
181+
" FreqSequence(sequence=[['b'], ['d']], freq=2),\n",
182+
" FreqSequence(sequence=[['b'], ['d'], ['c']], freq=2),\n",
183+
" FreqSequence(sequence=[['b'], ['f']], freq=2),\n",
184+
" FreqSequence(sequence=[['b', 'a']], freq=2),\n",
185+
" FreqSequence(sequence=[['b', 'a'], ['c']], freq=2),\n",
186+
" FreqSequence(sequence=[['b', 'a'], ['d']], freq=2),\n",
187+
" FreqSequence(sequence=[['b', 'a'], ['d'], ['c']], freq=2),\n",
188+
" FreqSequence(sequence=[['b', 'a'], ['f']], freq=2),\n",
189+
" FreqSequence(sequence=[['b', 'c']], freq=2),\n",
190+
" FreqSequence(sequence=[['b', 'c'], ['a']], freq=2),\n",
191+
" FreqSequence(sequence=[['c']], freq=4),\n",
192+
" FreqSequence(sequence=[['c'], ['a']], freq=2),\n",
193+
" FreqSequence(sequence=[['c'], ['b']], freq=3),\n",
194+
" FreqSequence(sequence=[['c'], ['c']], freq=3),\n",
195+
" FreqSequence(sequence=[['d']], freq=3),\n",
196+
" FreqSequence(sequence=[['d'], ['b']], freq=2),\n",
197+
" FreqSequence(sequence=[['d'], ['c']], freq=3),\n",
198+
" FreqSequence(sequence=[['d'], ['c'], ['b']], freq=2),\n",
199+
" FreqSequence(sequence=[['e']], freq=3),\n",
200+
" FreqSequence(sequence=[['e'], ['a']], freq=2),\n",
201+
" FreqSequence(sequence=[['e'], ['a'], ['b']], freq=2),\n",
202+
" FreqSequence(sequence=[['e'], ['a'], ['c']], freq=2),\n",
203+
" FreqSequence(sequence=[['e'], ['a'], ['c'], ['b']], freq=2),\n",
204+
" FreqSequence(sequence=[['e'], ['b']], freq=2),\n",
205+
" FreqSequence(sequence=[['e'], ['b'], ['c']], freq=2),\n",
206+
" FreqSequence(sequence=[['e'], ['c']], freq=2),\n",
207+
" FreqSequence(sequence=[['e'], ['c'], ['b']], freq=2),\n",
208+
" FreqSequence(sequence=[['e'], ['f']], freq=2),\n",
209+
" FreqSequence(sequence=[['e'], ['f'], ['b']], freq=2),\n",
210+
" FreqSequence(sequence=[['e'], ['f'], ['c']], freq=2),\n",
211+
" FreqSequence(sequence=[['e'], ['f'], ['c'], ['b']], freq=2),\n",
212+
" FreqSequence(sequence=[['f']], freq=3),\n",
213+
" FreqSequence(sequence=[['f'], ['b']], freq=2),\n",
214+
" FreqSequence(sequence=[['f'], ['b'], ['c']], freq=2),\n",
215+
" FreqSequence(sequence=[['f'], ['c']], freq=2),\n",
216+
" FreqSequence(sequence=[['f'], ['c'], ['b']], freq=2)]"
217+
]
218+
},
219+
"execution_count": 6,
220+
"metadata": {},
221+
"output_type": "execute_result"
222+
}
223+
],
224+
"source": [
225+
"sorted(model.freqSequences().collect())"
226+
]
227+
},
228+
{
229+
"cell_type": "code",
230+
"execution_count": null,
231+
"metadata": {},
232+
"outputs": [],
233+
"source": []
234+
}
235+
],
236+
"metadata": {
237+
"kernelspec": {
238+
"display_name": "Python 3",
239+
"language": "python",
240+
"name": "python3"
241+
},
242+
"language_info": {
243+
"codemirror_mode": {
244+
"name": "ipython",
245+
"version": 3
246+
},
247+
"file_extension": ".py",
248+
"mimetype": "text/x-python",
249+
"name": "python",
250+
"nbconvert_exporter": "python",
251+
"pygments_lexer": "ipython3",
252+
"version": "3.6.4"
253+
}
254+
},
255+
"nbformat": 4,
256+
"nbformat_minor": 2
257+
}

‎readme.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ http://www.cnblogs.com/pinard 刘建平Pinard
132132
[Apriori算法原理总结](https://www.cnblogs.com/pinard/p/6293298.html)|无
133133
[FP Tree算法原理总结](https://www.cnblogs.com/pinard/p/6307064.html)|无
134134
[PrefixSpan算法原理总结](https://www.cnblogs.com/pinard/p/6323182.html)|无
135-
[用Spark学习FP Tree算法和PrefixSpan算法](https://www.cnblogs.com/pinard/p/6340162.html)| to be uploaded
135+
[用Spark学习FP Tree算法和PrefixSpan算法](https://www.cnblogs.com/pinard/p/6340162.html)| [代码](https://github.com/ljpzzz/machinelearning/blob/master/classic-machine-learning/fp_tree_prefixspan.ipynb)
136136
[日志和告警数据挖掘经验谈](https://www.cnblogs.com/pinard/p/6039099.html) | 无
137137

138138
License MIT.

0 commit comments

Comments
 (0)