Hands-on Practice

by Suhyun Kim and Seungtae Park
iSystems Design Lab
http://isystems.unist.ac.kr/
UNIST

Table of Contents

In [1]:
import seaborn as sns; sns.set(style="ticks", color_codes=True)

1. Data Load

  • cPickle을 이용하여 데이터 로드
  • 예제 데이터는 아래 링크를 통해 받을 수 있습니다.
In [2]:
from six.moves import cPickle
In [3]:
data = cPickle.load(open('./data_files/rotation_data.pkl', 'rb'))
  • Data 구조 파악 (Dictionary로 저장해 둔 데이터)
In [4]:
print(data.keys())
dict_keys(['rubbing', 'unbal', 'misalign', 'normal'])
In [5]:
(X1, y1) = data['misalign']
(X2, y2) = data['unbal']
(X3, y3) = data['normal']
(X4, y4) = data['rubbing']
In [6]:
X1.shape
Out[6]:
(300, 820)
  • Plot 으로 확인하기
In [7]:
import matplotlib.pyplot as plt
In [8]:
for data_type in ['misalign', 'unbal', 'normal', 'rubbing']:
    X, y = data[data_type]
    plt.plot(X[10,:], label=data_type)
plt.legend(loc = 'lower right')
plt.xlim(0,X1.shape[1])
plt.show()

2. Preprocess

  • 편의를 위하여 데이터를 하나로 병합하기
In [9]:
import numpy as np
In [10]:
X = np.vstack([X1, X2, X3, X4])
y = np.concatenate([y1, y2, y3, y4])
  • 평균을 0으로 맞추기
  • 표준편차를 1로 맞추기
  • sklearn.preprocessing 에 StandardScaler 이용
In [11]:
from sklearn.preprocessing import StandardScaler
In [12]:
sdc = StandardScaler()
X = sdc.fit_transform(X)
In [13]:
plt.plot(X[100,:])
plt.xlim(0,X.shape[1])
plt.show()
  • PCA를 이용하여 feature 추출하기
  • 2차원으로 차원 축소
  • sklearn.decomposition 에 PCA 이용
In [14]:
from sklearn.decomposition import PCA
In [15]:
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)
In [16]:
print(X_pca.shape)
(1350, 2)
In [17]:
pca.explained_variance_ratio_
Out[17]:
array([ 0.11915963,  0.11747344])
  • 차원 축소 결과 확인
In [18]:
fig = plt.figure(figsize=(10,6))
color = ['r.', 'g.', 'b.', 'm.']
for i in range(4):
    ax = fig.add_subplot(1, 4, i+1, title='Class'+str(i))
    ax.plot(X_pca[y==i, 0], X_pca[y==i, 1], color[i])
    plt.axis('equal')
plt.show()
In [19]:
fig = plt.figure(figsize=(10,6))
color = ['r.', 'g.', 'b.', 'm.']
for i in range(4):
    plt.plot(X_pca[y==i, 0], X_pca[y==i, 1], color[i], label='Class'+str(i))
    plt.axis('equal')
plt.legend()
plt.show()

3. Data Split

  • Training data와 validation data로 나누기
  • sklearn.cross_validation에 StratifiedKFold 이용
In [20]:
from sklearn.cross_validation import StratifiedKFold
/mnt/disk1/suhyun/shvenv/lib/python3.4/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
In [21]:
skf = StratifiedKFold(y, n_folds=5)
In [22]:
train_indices, valid_indices = next(iter(skf))
X_train, y_train = X_pca[train_indices, :], y[train_indices]
X_valid, y_valid = X_pca[valid_indices, :], y[valid_indices]

RNN의 경우 label 을 다르게 하는 것이 편리하다

  • From
    • y = 1, ... , 4
  • To
    • y = [1, 0, 0, 0], ... , [0, 0, 0, 1]
  • sklearn.preprocessing에 OneHotEncoder 이용
In [23]:
from sklearn.preprocessing import OneHotEncoder
In [24]:
# n_values sets number of classes 
ohe = OneHotEncoder(n_values = 4)
y_to_ohg = y.reshape(-1, 1)
ohe.fit(y_to_ohg) # X1 is defined in LabelEncoder part
y_ohe = ohe.transform(y_to_ohg) # transform gives scipy.sparse type output
y_ohe = y_ohe.toarray()
  • RNN을 위한 X, y의 경우 끝에 _rnn을 붙임
    • X의 경우 차원 축소 되기 전 (PCA 적용 전 X)
In [25]:
X_train_rnn, y_train_rnn = X[train_indices, :], y_ohe[train_indices]
X_valid_rnn, y_valid_rnn = X[valid_indices, :], y_ohe[valid_indices]
In [26]:
X_train_rnn.shape
Out[26]:
(1080, 820)
In [27]:
y_train_rnn.shape
Out[27]:
(1080, 4)

4. Classification

  • Logistic Regression
  • Linear SVM
  • Nonlinear SVM
  • Recurrent Neural NetworK

4.1. Logistic Regression

  • Logistic Regression 모듈 import
In [28]:
from sklearn.linear_model import LogisticRegression
In [29]:
clf = LogisticRegression(C=1.0)
clf.fit(X_train, y_train)
Out[29]:
LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,
          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,
          verbose=0, warm_start=False)
  • Validation set 으로 성능 확인
In [30]:
pred = clf.predict(X_valid)
accuracy = np.mean(np.equal(pred, y_valid))
print('Accuracy = {:.4f}'.format(accuracy))
Accuracy = 0.4444

4.2. Linear SVM

  • SVC 모듈 import
In [31]:
from sklearn.svm import SVC
In [32]:
clf = SVC(kernel='linear', C=1.0)
clf.fit(X_train, y_train)
Out[32]:
SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape=None, degree=3, gamma='auto', kernel='linear',
  max_iter=-1, probability=False, random_state=None, shrinking=True,
  tol=0.001, verbose=False)
  • Validatioin set 으로 성능 확인
In [33]:
pred = clf.predict(X_valid)
accuracy = np.mean(np.equal(pred, y_valid))
print('Accuracy = {:.4f}'.format(accuracy))
Accuracy = 0.4444

4.3. Nonlinear SVM

  • SVC 모듈 import
In [34]:
from sklearn.svm import SVC
In [35]:
clf = SVC(kernel='rbf', C=1.0)
clf.fit(X_train,  y_train)
Out[35]:
SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',
  max_iter=-1, probability=False, random_state=None, shrinking=True,
  tol=0.001, verbose=False)
  • Validation set 으로 성능 확인
In [36]:
pred = clf.predict(X_valid)
accuracy = np.mean(np.equal(pred, y_valid))
print('Accuracy = {:.4f}'.format(accuracy))
Accuracy = 0.6407
In [37]:
svc = SVC(kernel='linear', C=1).fit(X_pca, y)
rbf_svc = SVC(kernel='rbf', C=1, gamma=1).fit(X_pca, y)
In [38]:
# create a mesh to plot in
h = 1  # step size in the mesh
x_min, x_max = X_pca[:, 0].min() - 1, X_pca[:, 0].max() + 1
y_min, y_max = X_pca[:, 1].min() - 1, X_pca[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))
In [39]:
# title for the plots
titles = ['Linear Model', 'Nonlinear Model']

fig = plt.figure(figsize=(15,15))
for i, clf in enumerate((svc, rbf_svc)):
    plt.subplot(2, 2, i + 1)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.coolwarm, alpha=0.8)

    # Plot also the training points
    # plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.coolwarm)
    plt.xlabel('X1')
    plt.ylabel('X2')
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.xticks(())
    plt.yticks(())
    plt.title(titles[i], fontsize=20)

plt.show()

4.4. Recurrent Neural Network

In [40]:
import tensorflow as tf
  • Define Variable
In [41]:
'''
batch size
learning rate
n_iter
flag
등 학습과정에서 필요할 variable 정의
'''
learning_rate = 0.5
n_iter = 5000
flag = 500
  • Define Network Size

    • input data의 shape 확인
In [42]:
'''
total length
input sequence
input steps
hidden layer 개수
등 네트워크를 구성하는데 직접적인 영향을 주는 variable정의
'''
total_length = 820
n_steps = 10
n_input_sequence = int(total_length//n_steps)
n_hidden = 30
n_classes = 4
  • Define Weights
In [43]:
'''
Network 안에 들어갈 weigth, biases 정의
'''

weights = {
    'out' : tf.Variable(tf.random_normal([n_hidden, n_classes], stddev = 0.1))
}

biases = {
    'out' : tf.Variable(tf.random_normal([n_classes], stddev = 0.1))
}
  • Define Network
In [44]:
def RNN(x, weights, biases, n_input, n_step, batch_size):
    x = tf.transpose(x, [1, 0, 2])
    x = tf.reshape(x, [-1, n_input])
    x = tf.split(x, n_step, 0)
    
    with tf.variable_scope('rnn') as scope : 
        lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)
        ch_tuple = lstm_cell.zero_state(batch_size, dtype=tf.float32)
        
        c_list = []
        h_list = []
        
        for input_ in x:
            try : 
                h, ch_tuple = lstm_cell(input_, ch_tuple)
            except ValueError : 
                scope.reuse_variables()
                h, ch_tuple = lstm_cell(input_, ch_tuple)
        
            c_list.append(ch_tuple[0])
            h_list.append(ch_tuple[1])
            
    return tf.matmul(h_list[-1], weights['out']) + biases['out']
        
  • Define Cost
In [45]:
x = tf.placeholder(tf.float32, [None, n_steps, n_input_sequence])
y = tf.placeholder(tf.float32, [None, n_classes])

Batch_size_place_holder = tf.placeholder(tf.int32)
RNN_result = RNN(x, weights, biases, n_input_sequence, n_steps, Batch_size_place_holder)

cost = tf.reduce_mean(tf.square(RNN_result - y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
  • Optimize
In [46]:
X_train_rnn.shape
Out[46]:
(1080, 820)
In [51]:
# Optimize 
init = tf.global_variables_initializer()
sess = tf.Session()

sess.run(init)

# Training cycle
for epoch in range(n_iter):
    X_train_reshape = np.reshape(X_train_rnn, [-1, n_steps, n_input_sequence])
    sess.run(optimizer, feed_dict={x: X_train_reshape, y: y_train_rnn, Batch_size_place_holder: X_train_reshape.shape[0]})
    c = sess.run(cost, feed_dict={x: X_train_reshape,  y: y_train_rnn, Batch_size_place_holder: X_train_reshape.shape[0]})
    if epoch % flag == 0:
        print ("Iter : {}".format(epoch))
        print ("Cost : {}".format(c))
Iter : 0
Cost : 0.1960446685552597
Iter : 500
Cost : 0.053301259875297546
Iter : 1000
Cost : 0.04368560016155243
Iter : 1500
Cost : 0.041778355836868286
Iter : 2000
Cost : 0.0406017079949379
Iter : 2500
Cost : 0.03429705649614334
Iter : 3000
Cost : 0.02543618343770504
Iter : 3500
Cost : 0.0201957356184721
Iter : 4000
Cost : 0.015899494290351868
Iter : 4500
Cost : 0.013636562041938305
  • Validatioin set 으로 성능 확인
In [52]:
X_valid_reshape = np.reshape(X_valid_rnn, [-1, n_steps, n_input_sequence])
pred = sess.run(RNN_result, feed_dict={x:X_valid_reshape, Batch_size_place_holder: X_valid_reshape.shape[0]})
accuracy = np.mean(np.equal(np.argmax(pred, 1), np.argmax(y_valid_rnn, 1)))
print('Accuracy = {:.4f}'.format(accuracy))
Accuracy = 0.9889
In [ ]:
%%javascript
$.getScript('https://kmahelona.github.io/ipython_notebook_goodies/ipython_notebook_toc.js')