BruceFan's Blog

Stay hungry, stay foolish

0%

用Tensorflow Serving部署自己的模型

Google为了解决机器学习模型部署上线至生产环境,发布了Tensorflow Serving。本文主要通过部署一个手写数字识别的模型来介绍Tensorflow Serving的基本用法。

构建CNN模型及checkpoint保存方式

如果不需要Tensorflow Serving部署模型的话,大部分人会选择传统的checkpoint方式保存训练好的模型,下面先看一下这种传统的保存方式:
代码清单 mnist_test.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import numpy as np
import os
import cv2

# 屏蔽waring信息
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

"""------------------加载数据---------------------"""
# 载入数据
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
# 改变数据格式,为了能够输入卷积层
trX = trX.reshape(-1, 28, 28, 1) # -1表示不考虑输入图片的数量,1表示单通道
teX = teX.reshape(-1, 28, 28, 1)

"""------------------构建模型---------------------"""
# 定义输入输出的数据容器
X = tf.placeholder("float", [None, 28, 28, 1], name="X")
Y = tf.placeholder("float", [None, 10])


# 定义和初始化权重、dropout参数
def init_weights(shape):
return tf.Variable(tf.random_normal(shape, stddev=0.01))


w1 = init_weights([3, 3, 1, 32]) # 3X3的卷积核,获得32个特征
w2 = init_weights([3, 3, 32, 64]) # 3X3的卷积核,获得64个特征
w3 = init_weights([3, 3, 64, 128]) # 3X3的卷积核,获得128个特征
w4 = init_weights([128 * 4 * 4, 625]) # 从卷积层到全连层
w_o = init_weights([625, 10]) # 从全连层到输出层

p_keep_conv = tf.placeholder("float", name="p_keep_conv")
p_keep_hidden = tf.placeholder("float", name="p_keep_hidden")


# 定义模型
def create_model(X, w1, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden):
# 第一组卷积层和pooling层
conv1 = tf.nn.conv2d(X, w1, strides=[1, 1, 1, 1], padding='SAME')
conv1_out = tf.nn.relu(conv1)
pool1 = tf.nn.max_pool(conv1_out, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
pool1_out = tf.nn.dropout(pool1, p_keep_conv)

# 第二组卷积层和pooling层
conv2 = tf.nn.conv2d(pool1_out, w2, strides=[1, 1, 1, 1], padding='SAME')
conv2_out = tf.nn.relu(conv2)
pool2 = tf.nn.max_pool(conv2_out, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
pool2_out = tf.nn.dropout(pool2, p_keep_conv)

# 第三组卷积层和pooling层
conv3 = tf.nn.conv2d(pool2_out, w3, strides=[1, 1, 1, 1], padding='SAME')
conv3_out = tf.nn.relu(conv3)
pool3 = tf.nn.max_pool(conv3_out, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
pool3 = tf.reshape(pool3, [-1, w4.get_shape().as_list()[0]]) # 转化成一维的向量
pool3_out = tf.nn.dropout(pool3, p_keep_conv)

# 全连层
fully_layer = tf.matmul(pool3_out, w4)
fully_layer_out = tf.nn.relu(fully_layer)
fully_layer_out = tf.nn.dropout(fully_layer_out, p_keep_hidden)

# 输出层
out = tf.matmul(fully_layer_out, w_o)

return out


model = create_model(X, w1, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden)

# 定义代价函数、训练方法、预测操作
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=Y))
train_op = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cost)
predict_op = tf.argmax(model, 1, name="predict")

# 定义一个saver
saver=tf.train.Saver()

# 定义存储路径
ckpt_dir="./ckpt_dir"
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)

"""------------------训练模型或者加载模型进行测试---------------------"""
train_batch_size = 128 # 训练集的mini_batch_size=128
test_batch_size = 256 # 测试集中调用的batch_size=256
epoches = 5 # 迭代周期
with tf.Session() as sess:
"""-------训练模型--------"""
# 初始化所有变量
tf.global_variables_initializer().run()

# 训练操作
# for i in range(epoches):
# train_batch = zip(range(0, len(trX), train_batch_size),
# range(train_batch_size, len(trX) + 1, train_batch_size))
# for start, end in train_batch:
# sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end],
# p_keep_conv: 0.8, p_keep_hidden: 0.5})
# # 每个周期用测试集中随机抽出test_batch_size个图片进行测试
# test_indices = np.arange(len(teX)) # 返回一个array[0,1...len(teX)]
# np.random.shuffle(test_indices) # 打乱这个array
# test_indices = test_indices[0:test_batch_size]
#
# # 获取测试集test_batch_size章图片的的预测结果
# predict_result = sess.run(predict_op, feed_dict={X: teX[test_indices],
# p_keep_conv: 1.0,
# p_keep_hidden: 1.0})
# # 获取真实的标签值
# true_labels = np.argmax(teY[test_indices], axis=1)
#
# # 计算准确率
# accuracy = np.mean(true_labels == predict_result)
# print("epoch", i, ":", accuracy)
#
# # 保存模型
# saver.save(sess,ckpt_dir+"/model.ckpt",global_step=i)

"""-----加载模型,用导入的图片进行测试--------"""
# 载入图片
src = cv2.imread('./2.png')

# 将图片转化为28*28的灰度图
src = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)
dst = cv2.resize(src, (28, 28), interpolation=cv2.INTER_CUBIC)

# 将灰度图转化为1*784的能够输入的网络的数组
picture = np.zeros((28, 28))
for i in range(0, 28):
for j in range(0, 28):
picture[i][j] = (255 - dst[i][j])
picture = picture.reshape(1, 28, 28, 1)

# 载入模型
saver.restore(sess, ckpt_dir+"/model.ckpt-4")
# 进行预测
predict_result = sess.run(predict_op, feed_dict={X: picture,
p_keep_conv: 1.0,
p_keep_hidden: 1.0})
print("你导入的图片是:", predict_result[0])

mnist_test.py文件包含了模型定义、训练和保存、加载。其中注释掉的代码为模型训练部分,去掉注释可以进行模型训练,训练后的模型保存在ckpt_dir中:

1
2
3
4
5
$ ls ckpt_dir
checkpoint model.ckpt-1.data-00000-of-00001 model.ckpt-2.index model.ckpt-3.meta
model.ckpt-0.data-00000-of-00001 model.ckpt-1.index model.ckpt-2.meta model.ckpt-4.data-00000-of-00001
model.ckpt-0.index model.ckpt-1.meta model.ckpt-3.data-00000-of-00001 model.ckpt-4.index
model.ckpt-0.meta model.ckpt-2.data-00000-of-00001 model.ckpt-3.index model.ckpt-4.meta

用Saved Model方式保存训练好的模型

为了能用Tensorflow Serving部署,需要用SavedModel方式保存模型。可以重新训练,再用SavedModel方式保存,也可以加载预训练的模型,保存为SavedModel方式。具体操作是在载入模型之后加入如下代码:

1
2
3
4
5
# Saved Model
tf.saved_model.simple_save(sess,
'savedmodel/1',
inputs={"X":X, "p_keep_conv":p_keep_conv, "p_keep_hidden":p_keep_hidden},
outputs={"predict":predict_op})

第一个参数sess为当前会话;第二个参数为保存模型的路径;第三个参数是模型输入,即使用模型进行预测时feed_dict里的变量;第四个参数是模型输出,对应模型预测时的第一个参数。
把训练模型部分的代码注释掉,运行mnist_test.py,会将模型保存为Tensorflow Serving可以用的形式:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
$ ls savedmodel/1
saved_model.pb variables
# 查看模型的输入输出
$ saved_model_cli show --dir savedmodel/1 --all

MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:

signature_def['serving_default']:
The given SavedModel SignatureDef contains the following input(s):
inputs['X'] tensor_info:
dtype: DT_FLOAT
shape: (-1, 28, 28, 1)
name: X:0
inputs['p_keep_conv'] tensor_info:
dtype: DT_FLOAT
shape: unknown_rank
name: p_keep_conv:0
inputs['p_keep_hidden'] tensor_info:
dtype: DT_FLOAT
shape: unknown_rank
name: p_keep_hidden:0
The given SavedModel SignatureDef contains the following output(s):
outputs['predict'] tensor_info:
dtype: DT_INT64
shape: (-1)
name: predict:0
Method name is: tensorflow/serving/predict

saved_model_cli命令行工具在安装过tensorflow之后就有了。

用Tensorflow Serving部署模型

接下来就可以用Tensorflow Serving对模型进行部署了,Tensorflow Serving可以提供gRPCREST两种方式,gRPC使用8500端口,REST使用8501端口,通过-p选项来映射docker和本地的端口,前面的是本地端口,后面的是docker端口,--name选项指定docker容器的名称,--mount选项挂载文件系统到容器,-e选项设置环境变量,-t选项分配虚拟终端:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
$ sudo docker run -p 8502:8501 --name mnist_model --mount type=bind,source=/home/fanrong/tfserving/savedmodel,target=/models/mnist_test -e MODEL_NAME=mnist_test -t tensorflow/serving
2019-05-09 07:31:04.975302: I tensorflow_serving/model_servers/server.cc:82] Building single TensorFlow model file config: model_name: mnist_test model_base_path: /models/mnist_test
2019-05-09 07:31:04.975674: I tensorflow_serving/model_servers/server_core.cc:461] Adding/updating models.
2019-05-09 07:31:04.975744: I tensorflow_serving/model_servers/server_core.cc:558] (Re-)adding model: mnist_test
2019-05-09 07:31:05.076332: I tensorflow_serving/core/basic_manager.cc:739] Successfully reserved resources to load servable {name: mnist_test version: 1}
2019-05-09 07:31:05.076402: I tensorflow_serving/core/loader_harness.cc:66] Approving load for servable version {name: mnist_test version: 1}
2019-05-09 07:31:05.076447: I tensorflow_serving/core/loader_harness.cc:74] Loading servable version {name: mnist_test version: 1}
2019-05-09 07:31:05.076533: I external/org_tensorflow/tensorflow/contrib/session_bundle/bundle_shim.cc:363] Attempting to load native SavedModelBundle in bundle-shim from: /models/mnist_test/1
2019-05-09 07:31:05.076560: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:31] Reading SavedModel from: /models/mnist_test/1
2019-05-09 07:31:05.081699: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:54] Reading meta graph with tags { serve }
2019-05-09 07:31:05.109514: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:182] Restoring SavedModel bundle.
2019-05-09 07:31:05.146630: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:285] SavedModel load for tags { serve }; Status: success. Took 70041 microseconds.
2019-05-09 07:31:05.146734: I tensorflow_serving/servables/tensorflow/saved_model_warmup.cc:101] No warmup data file found at /models/mnist_test/1/assets.extra/tf_serving_warmup_requests
2019-05-09 07:31:05.147120: I tensorflow_serving/core/loader_harness.cc:86] Successfully loaded servable version {name: mnist_test version: 1}
2019-05-09 07:31:05.153768: I tensorflow_serving/model_servers/server.cc:313] Running gRPC ModelServer at 0.0.0.0:8500 ...
[warn] getaddrinfo: address family for nodename not supported
2019-05-09 07:31:05.158494: I tensorflow_serving/model_servers/server.cc:333] Exporting HTTP/REST API at:localhost:8501 ...
[evhttp_server.cc : 237] RAW: Entering the event loop ...

接下来要对上线的模型请求服务,需要编写客户端代码:
代码清单 client.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#coding:utf-8

import requests
import cv2
import numpy as np

URL = 'http://localhost:8502/v1/models/mnist_test:predict'

def main():
src = cv2.imread('./2.png')
src = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)
dst = cv2.resize(src, (28, 28), interpolation=cv2.INTER_CUBIC)
picture = np.zeros((28,28))
for i in range(0, 28):
for j in range(0, 28):
picture[i][j] = (255 - dst[i][j])
picture = picture.reshape(1, 28, 28, 1)
predict_request = '{"inputs":{"X":%s, "p_keep_conv":1.0, "p_keep_hidden":1.0}}' % picture.tolist()
response = requests.post(URL, data=predict_request)
response.raise_for_status()
print(response.json()['outputs'])

if __name__ == '__main__':
main()

这里需要注意的是输入参数中的X,X本身是一个numpy数组类型,但是参数只能以字符串形式传递,如果直接用nunpy数组,转换为字符串是这样的:

1
2
3
4
5
6
7
8
9
10
11
12
[[[[  0.]
[ 0.]
[ 0.]
[ 0.]
[ 0.]
...
[ 0.]
[ 0.]
[ 0.]
[ 0.]
[ 0.]
[ 0.]]]]

服务端接收到字符串以后无法处理,会报400 Client Error: Bad Request for url: http://localhost:8502/v1/models/mnist_test:predict
所以需要先把numpy数组转换为list类型,list类型转换为字符串是这样的:

1
[[[[0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], ...[0.0], [0.0], [0.0], [0.0], [0.0]]]]

服务端接收到字符串之后,可以识别为list类型,进行正常处理。
最后是这种效果:

1
2
$ python client.py
[2]