Skip to content

Commit 64217eb

Browse files
xiaohu2015xiaohu2015
authored
Add files via upload
1 parent c44b41b commit 64217eb

7 files changed

Lines changed: 964 additions & 0 deletions

File tree

‎models/cnn.py‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
"""
2+
Convolution neural network
3+
author: Ye Hu
4+
2016/12/15
5+
"""
6+
import numpy as np
7+
import tensorflow as tf
8+
import input_data
9+
from logisticRegression import LogisticRegression
10+
from mlp import HiddenLayer
11+
12+
class ConvLayer(object):
13+
"""
14+
A convolution layer
15+
"""
16+
def __init__(self, inpt, filter_shape, strides=(1, 1, 1, 1),
17+
padding="SAME", activation=tf.nn.relu, bias_setting=True):
18+
"""
19+
inpt: tf.Tensor, shape [n_examples, witdth, height, channels]
20+
filter_shape: list or tuple, [witdth, height. channels, filter_nums]
21+
strides: list or tuple, the step of filter
22+
padding:
23+
activation:
24+
bias_setting:
25+
"""
26+
self.input = inpt
27+
# initializes the filter
28+
self.W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), dtype=tf.float32)
29+
if bias_setting:
30+
self.b = tf.Variable(tf.truncated_normal(filter_shape[-1:], stddev=0.1),
31+
dtype=tf.float32)
32+
else:
33+
self.b = None
34+
conv_output = tf.nn.conv2d(self.input, filter=self.W, strides=strides,
35+
padding=padding)
36+
conv_output = conv_output + self.b if self.b is not None else conv_output
37+
# the output
38+
self.output = conv_output if activation is None else activation(conv_output)
39+
# the params
40+
self.params = [self.W, self.b] if self.b is not None else [self.W, ]
41+
42+
43+
class MaxPoolLayer(object):
44+
"""pool layer"""
45+
def __init__(self, inpt, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), padding="SAME"):
46+
"""
47+
"""
48+
self.input = inpt
49+
# the output
50+
self.output = tf.nn.max_pool(self.input, ksize=ksize, strides=strides, padding=padding)
51+
self.params = []
52+
53+
54+
class FlattenLayer(object):
55+
"""Flatten layer"""
56+
def __init__(self, inpt, shape):
57+
self.input = inpt
58+
self.output = tf.reshape(self.input, shape=shape)
59+
self.params = []
60+
61+
class DropoutLayer(object):
62+
"""Dropout layer"""
63+
def __init__(self, inpt, keep_prob):
64+
"""
65+
keep_prob: float (0, 1]
66+
"""
67+
self.keep_prob = tf.placeholder(tf.float32)
68+
self.input = inpt
69+
self.output = tf.nn.dropout(self.input, keep_prob=self.keep_prob)
70+
self.train_dicts = {self.keep_prob: keep_prob}
71+
self.pred_dicts = {self.keep_prob: 1.0}
72+
73+
if __name__ == "__main__":
74+
# mnist examples
75+
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
76+
# define input and output placehoders
77+
x = tf.placeholder(tf.float32, shape=[None, 784])
78+
y_ = tf.placeholder(tf.float32, shape=[None, 10])
79+
# reshape
80+
inpt = tf.reshape(x, shape=[-1, 28, 28, 1])
81+
82+
# create network
83+
# params for training
84+
# conv and pool layer0
85+
layer0_conv = ConvLayer(inpt, filter_shape=[5, 5, 1, 32], strides=[1, 1, 1, 1], activation=tf.nn.relu,
86+
padding="SAME") # [?, 28, 28, 32]
87+
layer0_pool = MaxPoolLayer(layer0_conv.output, ksize=[1, 2, 2, 1],
88+
strides=[1, 2, 2, 1]) # [?, 14, 14, 32]
89+
# conv and pool layer1
90+
layer1_conv = ConvLayer(layer0_pool.output, filter_shape=[5, 5, 32, 64], strides=[1, 1, 1, 1],
91+
activation=tf.nn.relu, padding="SAME") # [?, 14, 14, 64]
92+
layer1_pool = MaxPoolLayer(layer1_conv.output, ksize=[1, 2, 2, 1],
93+
strides=[1, 2, 2, 1]) # [?, 7, 7, 64]
94+
# flatten layer
95+
layer2_flatten = FlattenLayer(layer1_pool.output, shape=[-1, 7*7*64])
96+
# fully-connected layer
97+
layer3_fullyconn = HiddenLayer(layer2_flatten.output, n_in=7*7*64, n_out=256, activation=tf.nn.relu)
98+
# dropout layer
99+
layer3_dropout = DropoutLayer(layer3_fullyconn.output, keep_prob=0.5)
100+
# the output layer
101+
layer4_output = LogisticRegression(layer3_dropout.output, n_in=256, n_out=10)
102+
103+
# params for training
104+
params = layer0_conv.params + layer1_conv.params + layer3_fullyconn.params + layer4_output.params
105+
# train dicts for dropout
106+
train_dicts = layer3_dropout.train_dicts
107+
# prediction dicts for dropout
108+
pred_dicts = layer3_dropout.pred_dicts
109+
110+
# get cost
111+
cost = layer4_output.cost(y_)
112+
# accuracy
113+
accuracy = layer4_output.accuarcy(y_)
114+
predictor = layer4_output.y_pred
115+
# 定义训练器
116+
train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(
117+
cost, var_list=params)
118+
119+
# 初始化所有变量
120+
init = tf.global_variables_initializer()
121+
122+
# 定义训练参数
123+
training_epochs = 10
124+
batch_size = 100
125+
display_step = 1
126+
127+
# 开始训练
128+
print("Start to train...")
129+
with tf.Session() as sess:
130+
sess.run(init)
131+
for epoch in range(training_epochs):
132+
avg_cost = 0.0
133+
batch_num = int(mnist.train.num_examples / batch_size)
134+
for i in range(batch_num):
135+
x_batch, y_batch = mnist.train.next_batch(batch_size)
136+
# 训练
137+
train_dicts.update({x: x_batch, y_: y_batch})
138+
139+
sess.run(train_op, feed_dict=train_dicts)
140+
# 计算cost
141+
pred_dicts.update({x: x_batch, y_: y_batch})
142+
avg_cost += sess.run(cost, feed_dict=pred_dicts) / batch_num
143+
# 输出
144+
if epoch % display_step == 0:
145+
pred_dicts.update({x: mnist.validation.images,
146+
y_: mnist.validation.labels})
147+
val_acc = sess.run(accuracy, feed_dict=pred_dicts)
148+
print("Epoch {0} cost: {1}, validation accuacy: {2}".format(epoch,
149+
avg_cost, val_acc))
150+
151+
print("Finished!")
152+
test_x = mnist.test.images[:10]
153+
test_y = mnist.test.labels[:10]
154+
print("Ture lables:")
155+
print(" ", np.argmax(test_y, 1))
156+
print("Prediction:")
157+
pred_dicts.update({x: test_x})
158+
print(" ", sess.run(predictor, feed_dict=pred_dicts))
159+
tf.scan()
160+
161+
162+
163+
164+

‎models/da.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
"""
2+
Denoising Autoencoder (DA)
3+
author: Ye Hu
4+
2016/12/16
5+
"""
6+
import os
7+
import timeit
8+
9+
import numpy as np
10+
import tensorflow as tf
11+
from PIL import Image
12+
13+
import input_data
14+
from utils import tile_raster_images
15+
16+
17+
18+
class DA(object):
19+
"""A denoising autoencoder class (using tied weight)"""
20+
def __init__(self, inpt, n_visiable=784, n_hidden=500, W=None, bhid=None,
21+
bvis=None, activation=tf.nn.sigmoid):
22+
"""
23+
inpt: tf.Tensor, the input
24+
:param n_visiable: int, number of hidden units
25+
:param n_hidden: int, number of visable units
26+
:param W, bhid, bvis: tf.Tensor, the weight, bias tensor
27+
"""
28+
self.n_visiable = n_visiable
29+
self.n_hidden = n_hidden
30+
# initialize the weight and bias if not given
31+
if W is None:
32+
bound = -4*np.sqrt(6.0 / (self.n_hidden + self.n_visiable))
33+
W = tf.Variable(tf.random_uniform([self.n_visiable, self.n_hidden], minval=-bound,
34+
maxval=bound), dtype=tf.float32)
35+
if bhid is None:
36+
bhid = tf.Variable(tf.zeros([n_hidden,]), dtype=tf.float32)
37+
if bvis is None:
38+
bvis = tf.Variable(tf.zeros([n_visiable,]), dtype=tf.float32)
39+
self.W = W
40+
self.b = bhid
41+
# reconstruct params
42+
self.b_prime = bvis
43+
self.W_prime = tf.transpose(self.W)
44+
# keep track of input and params
45+
self.input = inpt
46+
self.params = [self.W, self.b, self.b_prime]
47+
# activation
48+
self.activation = activation
49+
50+
def get_encode_values(self, inpt):
51+
"""Compute the encode values"""
52+
return self.activation(tf.matmul(inpt, self.W) + self.b)
53+
54+
def get_decode_values(self, encode_input):
55+
"""Get the reconstructed values"""
56+
return self.activation(tf.matmul(encode_input, self.W_prime) + self.b_prime)
57+
58+
def get_corrupted_input(self, inpt, corruption_level):
59+
"""
60+
Randomly zero the element of input
61+
corruption_level: float, (0,1]
62+
"""
63+
# the shape of input
64+
input_shape = tf.shape(inpt)
65+
# the probablity for corruption
66+
probs = tf.tile(tf.log([[corruption_level, 1-corruption_level]]),
67+
multiples=[input_shape[0], 1])
68+
return tf.mul(tf.cast(tf.multinomial(probs, num_samples=input_shape[1]),
69+
dtype=tf.float32), inpt)
70+
71+
def get_cost(self, corruption_level=0.3):
72+
"""Get the cost for training"""
73+
corrupted_input = self.get_corrupted_input(self.input, corruption_level)
74+
encode_output = self.get_encode_values(corrupted_input)
75+
decode_output = self.get_decode_values(encode_output)
76+
# use cross_entropy
77+
cross = tf.mul(self.input, tf.log(decode_output)) + \
78+
tf.mul(1.0-self.input, tf.log(1.0-decode_output))
79+
cost = -tf.reduce_mean(tf.reduce_sum(cross, axis=1))
80+
return cost
81+
82+
if __name__ == "__main__":
83+
# mnist examples
84+
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
85+
# define input
86+
x = tf.placeholder(tf.float32, shape=[None, 784])
87+
# set random_seed
88+
tf.set_random_seed(seed=99999)
89+
# the DA model
90+
da = DA(x, n_visiable=784, n_hidden=500)
91+
# corruption level
92+
corruption_level = 0.0
93+
learning_rate = 0.1
94+
cost = da.get_cost(corruption_level)
95+
params = da.params
96+
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost, var_list=params)
97+
init = tf.global_variables_initializer()
98+
99+
output_folder = "dA_plots"
100+
if not os.path.isdir(output_folder):
101+
os.makedirs(output_folder)
102+
os.chdir(output_folder)
103+
104+
training_epochs = 10
105+
batch_size = 100
106+
display_step = 1
107+
print("Start training...")
108+
start_time = timeit.default_timer()
109+
with tf.Session() as sess:
110+
sess.run(init)
111+
for epoch in range(training_epochs):
112+
avg_cost = 0.0
113+
batch_num = int(mnist.train.num_examples / batch_size)
114+
for i in range(batch_num):
115+
x_batch, _ = mnist.train.next_batch(batch_size)
116+
# 训练
117+
sess.run(train_op, feed_dict={x: x_batch})
118+
# 计算cost
119+
avg_cost += sess.run(cost, feed_dict={x: x_batch,}) / batch_num
120+
# 输出
121+
if epoch % display_step == 0:
122+
print("Epoch {0} cost: {1}".format(epoch, avg_cost))
123+
124+
end_time = timeit.default_timer()
125+
training_time = end_time - start_time
126+
print("Finished!")
127+
print(" The {0}%% corruption code ran for {1}.".format(corruption_level*100, training_time/60,))
128+
W_value = sess.run(da.W_prime)
129+
image = Image.fromarray(tile_raster_images(
130+
X=W_value,
131+
img_shape=(28, 28), tile_shape=(10, 10),
132+
tile_spacing=(1, 1)))
133+
image.save('filters_corruption_{0}.png'.format(int(corruption_level*100)))
134+
135+
136+
137+

0 commit comments

Comments
 (0)