Tensorflow framework of deep learning realizes vgg/rnn network / verification code generation and recognition / text classification

① Realization VGG A network model

cwd = os.getcwd()           # Get the current path 
VGG_PATH = cwd + "/data/imagenet-vgg-verydeep-19.mat"
data = scipy.io.loadmat(VGG_PATH)
mean = data['normalization'][0][0][0]
mean_pixel = np.mean(mean, axis=(0,1))  # Get the average value of three channels 
weights= data['layers'][0]
# find   The dimension corresponding to the weight parameter 
print(weights[0][0][0][0][0][0].shape)
print(weights[0][0][0][0][0][1].shape)

def net(data_path, input_image):
    layers = (
      'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1'
      ...
      ...                                                       #19 Layer of the network 
    )
    data = scipy.io.loadmat(VGG_PATH)
    mean = data['normalization'][0][0][0]
    mean_pixel = np.mean(mean, axis=(0,1))  # Get the average value of three channels 
    weights= data['layers'][0]
    net = {}
    current = input_image
    for i, name in enumerate(layers):
        kind = name[:4]
        if kind == 'conv':
            kernels, bias = weights[i][0][0][0][0]         # representative w and b
            kernels = np.transpose(kernels, (1,0,2,3))    # Conversion parameter format 
            bias = bias.reshape(-1)
            current = _conv_layer(current, kernels, bias)
        elif kind == 'relu':
            current = tf.nn.relu(current)
        elif kind == 'pool':
            current = _pool_layer(current)
        net[name] = current             #name  The name of each layer , Store the value transmitted by each layer 
    assert len(net) ==len(layers)
print("Network for VGG ready")

def _conv_layer(input, weights, bias):
    conv = tf.nn.conv2d(input, tf.constant(weights), striders=(1,1,1,1), padding='SAME')
    return tf.nn.bias_add(conv, bias)
def _pool_layer(input):
    return tf.nn.max_pool(input, ksize=(1,2,2,1), strides=(1,2,2,1), padding='SAME')
def preprocess(image, mean_pixel):
    return image - mean_pixel
def unprocess(image, mean_pixel)
    return image + mean_pixel
def imread(path):
    return scripy.misc.imread(path).astype(np.float)
def imsave(path, img):
    img = np.clip(image, 0 ,255).astype(np.uint8)
    scripy.misc.imsave(path, img)
print("Functions for VGG ready")

VGG_PATH =cwd + "/ / .mat"
IMG_PATH = cwd + "/data/ .jpg"
input_image = imread(IMG_PATH)
shape = (1, input_image.shape[0], input_image.shape[1], input_image.shape[2])
with tf.Session() as sess:
    image = tf.placeholder('float', shape=shape)
    nets, mean_pixel, all_layers = net(VGG_PATH, image)
    input_image

② Realization RNN A network model

 Accept input ：
 serialize  b1--b2--b3--b4

 Example ： Handle MINIST Data sets 
 Overall data -》 Sequence data    
diminput = 28
dimhidden = 128
dimoutput = nclasses
nsteps = 28       # step 
weights = {
    'hidden':tf.Variable(tf.random_normal([diminput, dimhidden])),
    'out':tf.Variable(tf.random_normal([dimhidden, dimoutput]))
}
biases = {
    'hidden': tf.Variable(tf.random_normal([dimhidden]))
    'out':tf.Variable(tf.random_normal([dimoutput]))
}

def _RNN(_X, _W, _b, _nsteps, _name):
    #=>[nsteps, batchsize, diminput]
    _X = tf.transpose(_X, [1, 0, 2])
    _X = tf.reshape(_X, [-1, diminput])
    _H = tf.matmul(_X, _W['hidden']) + _b['hidden']
    _Hsplit = tf.split(0, _nsteps, _H)
    with tf.variable_scope(_name) as scope:
        scope.reuse_variables()        # Variable sharing 
        lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(dimhidden, forget_bias=1.0)
        _LSTM_0, _LSTM_S = tf.nn.rnn(lstm_cell, _Hsplit, dtype=tf.float32)
    # Output   Last position 
    _0 = tf.matmul(_LSTM_0[-1], _W['out']) + _b['out']
    #Return!
    return {
            'X':_X, 'H',_H, 'Hsplit':_Hsplit, 'LSTM_0':_LSTM_0, 'LSTM_S':_LSTM_S, '0':_0
}

learning_rate = 0.001
x = tf.placeholder("float", [None, nsteps, diminput])
y = tf.placeholder("float", [dimoutput])
myrnn = _RNN(x, weights, biases ,nsteps, 'basic')  #basic name ： Name the domain name 
pred = myrnn['0']
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
optm = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
accr =tf.reduce_mean(tf.cast(tf.equal(tf.argmax(pred,1), tf.argmax(y,1)), tf.float32))
init = tf.global_variables_initializer()
print("Network Ready!")

training_epochs = 5
batch_size = 16
display_step = 1
sess = tf.Session()
sess.run(init)
print("Start optimization")
for epoch in range(training_epochs):
    avg_cost = 0
    total_batch = 100
    for i in range(total_batch):
        batch_xs, batch_ys = mnist.train.next_batch(batch_size)
        batch_xs = batch_xs.reshape((batch_size, nsteps, diminput))
        feeds = {x:batch_xs, y:batch_ys}
        sess.run(optm, feed_dict=feeds)
        avg_cost +=sess.run(cost, feed_dict=feeds)/ total_batch
    if epoch%display_step == 0:
        print("Epoch: %03d%03d"%(epoch, training_epochs, avg_cost))
        feeds ={x:batch_xs, y:batch_ys}
        train_acc = sess.run(accr, feed_dict=feeds)
        print("Training accuracy:%.3f"%(train_acc))
        testimgs = testimgs.reshape((ntest, nsteps, diminput))
        feeds ={x:testimgs, y:testlabels, istate:np.zeros((ntest, 2*dimhidden))}
        test_acc = sess.run(accr, feed_dict=feeds)
        print("Test accuracy:%.3f"%(test_acc))
print("Optimization Finished.")

③ Verification code generation and recognition

from captcha.image import ImageCaptcha  # Verification Code 
from PIL import Image
import matplotlib.pyplot as plt
import random 
import numpy as np
import tensorflow as tf

number = ['0', '1', '2', '3', '4', '5', '6','7','8','9']
alphabet = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'g', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's' ,'t', 'u', 'v', 'w', 'x', 'y', 'z']
ALPHABET=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'G', 'K', 'L', 'M', 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z']

# Generate string function 
def random_captcha_text(char_set = number+alphabet+ALPHABET, captcha_size=4):
    captcha_text = []
    for i in range(captcha_size):
        c=random.choice(char_set)
        captcha_text.append(c)
    return captcha_text
    
# Generate image function 
def gen_captcha_text_and_image():
    image = ImageCaptcha()
    captcha_text = random_captcha_text()
    captcha_text = ''.join(captcha_text)
    captcha = image.generate(captcha_text)   # Generate pictures 
    captcha_image = Image.open(captcha)
    captcha_image = np.array(captcha_image) # Convert to 3D pixels 
    return captcha_text, captcha_image
    
# Show the generated picture 
if __name__ =='__main__':
    text, image = gen_captcha_text_and_image()
    f=plt.figure()
    ax = f.add_subplot(111)
    ax.text(0.1, 0.9, text, ha='center', va='center', transform=ax.transAxes)
    plt.imshow(image)
    plt.show()
    
if __name__ == '__main__':
    train = 0
    # Training network 
    if train == 0:
        number = ['0', '1', '2', '3',...]
        text, image = gen_captcha_text_and_image()
        print(' Captcha image channel:', image.shape)
        #  Image size 
        IMAGE_HEIGHT = 60
        IMAGE_WIDTH = 160
        MAX_CAPTCHA = len(text)
        print(" The maximum number of characters in the verification code text ", MAX_CAPTCHA)
        # The amount of text turning 
        char_set = number + alphabet + ALPHABET
        CHAR_SET_LEN = len(char_set)
        X=tf.placeholder(tf.float32, [None, IMAGE_HEIGHT*IMAGE_WIDTH]) # One row represents one data 
        Y=tf.placeholder(tf.float32, [None, MAX_CAPTCHA*CHAR_SET_LEN]) # Each individual represents    example   Numbers 2  use 0010000000 This format represents 
        keep_prob = tf.placeholder(tf.float32)   #dropout Retention rate 
        train_crack_captcha_cnn()
    # Testing phase 
    if train == 1:
        number = ['0', '1', '2', '3'...]
        IMAGE_HEIGHT = 60
        IMAGE_WIDTH = 160
        #  Validation function 
        crack_captcha(captcha_image)
        
# Network structure        
def train_crack_captcha_cnn():
    output = crack_captcha_cnn()  # framework 
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=Y))
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(loss)
    predict = tf.reshape(output, [-1, MAX_CAPTCHA, CHAR_SET_LEN])
    max_idx_p = tf.argmax(predict, 2)
    max_idx_l = tf.argmax(tf.reshape(Y, [-1, MAX_CAPTCHA, CHAR_SET_LEN]), 2)
    correct_pred = tf.equal(max_idx_p, max_idx_l)           
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
    saver = tf.train.Saver()
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        step = 0
        while True:
            batch_x, batch_y = get_next_batch(64)
            _, loss_ = sess.run([optimizer, loss], feed_dict={X:batch_x, Y:batch_y, keep_prob:0.75})
            if step % 100 == 0:
                batch_x_test, batch_y_test = get_next_batch(100)
                acc = sess.run(accuracy, feed_dict={X:batch_x_test, Y:batch_y_test, keep_prob:0.75})
                print(step, acc)
                if acc > 0.85:
                    saver.save(sess, "./model/crack.model", global_step = step)
                    break
            step+=1
            
#  Definition CNN
def crack_captcha_cnn(w_alpha=0.01, b_alpha=0.1):
    x = tf.reshape(X, shape=[-1, IMAGE_HEIGHT, IMAGE_WIDTH, 1])
    # 3 conv layer
    w_c1 = tf.Variable(w_alpha*tf.random_normal([3, 3, 1, 32]))  #1 Is the input channel  32 Is the output characteristic diagram 
    b_c1 = tf.Variable(b_alpha*tf.random_normal([32]))
    conv1 = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(x, w_c1, strides=[1, 1, 1, 1],padding='SAME'), b_c1))
    conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1,2,2,1], padding='SAME')  #2*2 The window for pooling operation 
    conv1 = tf.nn.dropout(conv1, keep_prob)   

    w_c2 = tf.Variable(w_alpha*tf.random_normal([3, 3, 32, 64]))
    b_c2 = tf.Variable(b_alpha*tf.random_normal([64]))
    conv2 = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(conv1, w_c2, strides=[1, 1, 1, 1],padding='SAME'), b_c2))
    conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1,2,2,1], padding='SAME')
    conv2 = tf.nn.dropout(conv2, keep_prob)   

    w_c3 = tf.Variable(w_alpha*tf.random_normal([3, 3, 64, 64]))
    b_c3 = tf.Variable(b_alpha*tf.random_normal([64]))
    conv3 = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(conv2, w_c3, strides=[1, 1, 1, 1],padding='SAME'), b_c3))
    conv3 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1], strides=[1,2,2,1], padding='SAME')
    conv3 = tf.nn.dropout(conv3, keep_prob) 

    #  Fully connected layer 
    w_d = tf.Variable(w_alpha*tf.random_normal([8*20*64, 1024])  #8*20 It needs to be calculated manually  60--30--15--8( Three pooling operations )
    b_d = tf. Variable(b_alpha*tf.random_normal([1024]))
    dense = tf.reshape(conv3, [-1, w_d.get_shape().as_list()[0]])
    dense = tf.nn.relu(tf.add(tf.matmul(dense, w_d), b_d))
    dense = tf.nn.dropout(dense, keep_prob)

    w_out = tf.Variable(w_alpha*tf.random_normal([1024,MAX_CAPTCHA*CHAR_SET_LEN]))
    b_out = tf.Variable(b_alpha*tf.random_normal([MAX_CAPTCHA*CHAR_SET_LEN]))
    out = tf.add(tf.matmul(dense,w_out), b_out)
    return out
      
# Generate a workout batch
def get_next_batch(batch_size=128):
    batch_x = np.zeros([batch_size, IMAGE_HEIGHT*IMAGE_WIDTH])
    batch_y = np.zeros([batch_size, MAX_CAPTCHA*CHAR_SET_LEN])
    def wrap_gen_captcha_text_and_image():
        while True:
            text, image = gen_captcha_text_and_image()
            if image.shape == (60, 160, 3):
                return text, image
    for i in range(batch_size):
        text, image = wrap_gen_captcha_text_and_image()
        image = convert2gray(image)
        batch_x[i, :] = image.flatten() / 255
        batch_y[i, :] = text2vec(text)
    return batch_x, batch_y


def crack_captcha(captcha_image):
    out_put = crack_captcha_cnn()
    saver = tf.train.Saver()
    with tf.Session() as sess:
        saver.restore(sess, "./model/.model")
        predict = tf.argmax(tf.reshape(out_put, [-1, MAX_CAPTCHA, CHAR_SET_LEN]), 2)
        text_list = sess.run(predict, feed_dict={x:[captcha_image], keep_prob:1})
        text = text_list[0].tolist()
        return text

#  String steering function 
def text2vec(text):
    text_len = len(text)
    if text_len > MAX_CAPTCHA:
        raise ValueError(' The verification code is the longest %d Characters '%(MAX_CAPTCHA))
    vector = np.zeros(MAX_CAPTCHA*CHAR_SET_LEN)
    def char2pos(c):
        if c == '_':
            k = 62
            return k           # 26+26+10=62
        k = ord(c) - 48        # ASCII code 48 by 0
        if k > 9:
            k = ord(c) - 55     #ASCII code A by 65  So it's -65+10
            if k > 35:
                k = ord(c) - 61   #ASCII code a by 97  So it's -97+10+26
                if k > 61:
                    raise ValueError('No Map')
        return k
    for i, c in enumerate(text):
        idx = i * CHAR_SET_LEN + char2pos(c)
        vector[idx] = 1
    return vector

#  Color image to gray image function （ Color is not useful for identifying verification codes ）
def convert2gray(img):
    if len(img.shape) > 2:
        gray = np.mean(img, -1)
        #  The normal conversion method is as follows ：
        # r, g, b = img[:,:,0], img[:,:,1], img[:,:,2]
        # gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
        return gray
    else:
        return img

④ Realize text classification
file 1 train.py

import tensorflow as tf
import numpy as np
import os
import time
import datetime
import data_helpers
from text_cnn import TextCNN
from tensorflow.contrib import learn

# Data loading parameters
tf.flags.DEFINE_float("dev_sample_percentage", .1, "Percentage of the training data to use for validation")
#tf.flags.DEFINE_string("positive_data_file", "./data/rt-polaritydata/rt-polarity.pos", "Data source for the positive data.")
#tf.flags.DEFINE_string("negative_data_file", "./data/rt-polaritydata/rt-polarity.neg", "Data source for the negative data.")
tf.flags.DEFINE_string("positive_data_file", "./data/ham_5000.utf8", "Data source for the positive data.")
tf.flags.DEFINE_string("negative_data_file", "./data/spam_5000.utf8", "Data source for the negative data.")
tf.flags.DEFINE_integer("num_labels", 2, "Number of labels for data. (default: 2)")

# Model hyperparameters
tf.flags.DEFINE_integer("embedding_dim", 128, "Dimensionality of character embedding (default: 128)")
tf.flags.DEFINE_string("filter_sizes", "3,4,5", "Comma-spearated filter sizes (default: '3,4,5')")
tf.flags.DEFINE_integer("num_filters", 128, "Number of filters per filter size (default: 128)")
tf.flags.DEFINE_float("dropout_keep_prob", 0.5, "Dropout keep probability (default: 0.5)")
tf.flags.DEFINE_float("l2_reg_lambda", 0.0, "L2 regularization lambda (default: 0.0)")

# Training paramters
tf.flags.DEFINE_integer("batch_size", 64, "Batch Size (default: 64)")
tf.flags.DEFINE_integer("num_epochs", 200, "Number of training epochs (default: 200)")
tf.flags.DEFINE_integer("evaluate_every", 100, "Evalue model on dev set after this many steps (default: 100)")
tf.flags.DEFINE_integer("checkpoint_every", 100, "Save model after this many steps (defult: 100)")
tf.flags.DEFINE_integer("num_checkpoints", 5, "Number of checkpoints to store (default: 5)")

# Misc parameters
tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")

FLAGS=tf.flags.FLAGS
FLAGS._pares_flags()
print('\nParameters:')
for attr,value in sorted(FLAGS._flags.items()):
    print('{}={}').format(attr.upper(), value)
x_text, y = data_helpers.load_data_and_labels(FLAG.positive_data_file, FLAG.negative_data_file)

max_document_length = max(len(x.split(' ')) for x in x_text)
vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)
x = np.array( vacab_processor.fit_transform(x_text) )
np.random.seed(10)
shuffle_indices = np.random.permytation(np.arrange(len(y)))
x_shuffled = x[shuffle_indices]
y_shuffled = y[shuffle_indices]

dev_sample_index = -1*int(FLAGS.dev_sample_percetage*float(len(y)))
x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]
y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]

with tf.Graph().as_default():
    session_conf = tf.ConfigProto( allow_soft_placement = FLAGS.allow_soft_placement,
                                                   log_device_placement = FLAGS.log_device_placement
)  # Designated equipment , Print log 
    sess  = tf.Session(config = session_conf)
    with sess.as_default():
        cnn = TextCNN(
                        sequence_length =x_train.shape[1],
                        num_classes = y_train.shape[1],
                        vocab_size = len(vocab_processor.vocabulary_),
                        embedding_size = FLAGS.embedding,
                        filter_size = list(map(int, FLAGS.filter_size.split(',')))
                        num_filters = FLAGS.num_filters,
                        l2_reg_lambda =FLAGS.L2_reg_lambda
)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints)
sess.run(tf.global_variables_initializer())
batches = data_helpers.batch_iter(list(zip(x_train, y_train)）, FLAGS.batch_size, FLAGS.num_epochs)

    global_step = tf.Variable(0, name='global_step')
    optimizer = tf.train.AdamOptimizer(le-3)
    grads_and_vars = optimizer.compute_gradients(cnn.loss)
    train_op = optimizer.apply

def train_step(x_batch, y_batch):
 """
A single training step
 """
    feed_dict = {
              cnn.input_x: x_batch,
              cnn.input_y: y_batch,
              cnn.dropout_keep_prob: FLAGS.dropout_keep_prob
            }
    _, step, summaries, loss, accuracy = sess.run(
                [train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy],
                feed_dict)
   time_str = datetime.datetime.now().isoformat()
   print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
   train_summary_writer.add_summary(summaries, step)

def dev_step(x_batch, y_batch, writer=None):
"""
Evaluates model on a dev set
"""
    feed_dict = {
              cnn.input_x: x_batch,
              cnn.input_y: y_batch,
              cnn.dropout_keep_prob: 1.0
            }
    step, summaries, loss, accuracy = sess.run(
                [global_step, dev_summary_op, cnn.loss, cnn.accuracy],
                feed_dict)
    time_str = datetime.datetime.now().isoformat()
    print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
    if writer:
        writer.add_summary(summaries, step)
        
# Training loop. For each batch...
        for batch in batches:
            x_batch, y_batch = zip(*batch)
            train_step(x_batch, y_batch)
            current_step = tf.train.global_step(sess, global_step)
            if current_step % FLAGS.evaluate_every == 0:
                print("\nEvaluation:")
                dev_step(x_dev, y_dev, writer=dev_summary_writer)
                print("")
            if current_step % FLAGS.checkpoint_every == 0:
                path = saver.save(sess, checkpoint_prefix, global_step=current_step)
                print("Saved model checkpoint to {}\n".format(path))

file 2 data_helpers.py

#
def load_data_and_labels(positive_data_file, negative_data_file):
    positive = open(positive_data_file, 'rb').read().decode('utf-8')
    negative = open(negative_data_file, 'rb').read().decode('utf-8')
    positive_examples = positive.split('\n')[:-1]
    negative_examples = negative.split('\n')[:-1]
    positive_examples = [s.strip() for s in positive_examples]
    negative_examples = [s.strip() for s in negative_examples]
    x_text = positive_examples + negative_examples
    #  Data cleaning  x_text = [clean_str(sent) for sent in x_text]

    positive_label = [[0, 1] for _ in positive_examples]
    negative_label = [[1, 0] for _ in negative_examples]
    y = np.concatence([positive_label, negative_label], 0)
    return [x_text, y]
 
 def batch_iter(data, batch_size, num_epoch, shuffle=True):
    data = np.array(data)
    data_size = len(data)
    num_batcheds_per_epoch = int (len(data)-1)/batch_size + 1
    for epoch in range(num_epoch):
        if shuffle:
            shuffle_indices = np.random.permutation(np.arrange(data_size))
            shuffle_data = data[shuffle_indices]
        else:
            shuffle_data = data
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num*batch_size
            end_index = min((batch_num+1)*batch_size,data_size)
            yield shuffle_data[start_index:end_index]

file 3 text_cnn.py

import tensorflow as tf

class TextCNN(object):
    def __init__(self, sequence_length, num_classes, vocab_size, embedding_size, filter_sizes, num_filters, l2_reg_lambda):
        self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name='input_x')
        self.input_y = tf.placeholder(tf.float32, [None, num_classes], name='input_y')
        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
        l2_loss = tf.constant(0.0)
        with tf.device('/cpu:0'), tf.name_scope('embedding'):
            self.W = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0, name='W'))
            self.embedded_chars = tf.nn.embedding_lookup(self.W, self.input_x)
            self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)
        pooled_outputs = []
        #  The cycle is different filter Size 
        for i, filter_size in enumerate(filter_sizes):
            with tf.name.scope('conv-maxpool-%s' % filter_size):
                filter_shape = [filter_size, embedding_size, 1, num_filters]
                W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name='W')
                b = tf.Variable(tf.constant(0.1), shape=[num_filters], name='b')
                conv = tf.nn.conv2d(self.embedded_chars_expanded, W, strides=[1, 1, 1, 1], padding='VALID', name='conv')
                h = tf.nn.relu(tf.nn.bias_add(conv, b), name='relu')
                pooled = tf.nn.max_pool(h, ksize=[1, sequence_length - filter_sizes + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID', name='pool')
                pooled_outputs.append(pooled)
        num_filters_total = num_filters * len(filter_sizes)
        self.h_pool = tf.concat(3, pooled_outputs)
        self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])

        with tf.name.scope('dropout'):
            self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)
        # Output layer 
        with tf.name.scope('output'):
            W = tf.get_variable('W', shape=[num_filters_total, num_classes], initializer = tf.contrib.layers.Xavier_initializer())
            b = tf.Variable(tf.constant(0.1), shape=[num_classes], name='b')
            l2_loss += tf.nn.l2_loss(W)
            l2_loss += tf.nn.l2_loss(b)
            self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name='score')
            self.predictions = tf.argmax(self.scores, 1, name="predictions")
        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy(logits=self.scores, labels=self.input_y)
            self.loss = tf.reduce_mean(losses) + l2_loss * l2_reg_lambda
        with tf.name_scope('accuracy'):
            correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y), 1)
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions), 'float')