[214] Ai Serving Platform: 하루 수 억 건의 인퍼런스를 처리하기 위한 고군분투기

5. ...
// 여기쯤에 인퍼런스 엔진(1)을 메모리에 올리고
...
// 웹서버를 하나 띄워서 (1)을 돌리자
httpd = SocketServer.TCPServer(
(“”, Port), handler)
httpd.serve_forever()
...

8. •
•
•
•
•
•

12. # Install hdfscli for downloading model files.
RUN pip install hdfs
COPY files/.hdfscli.cfg /root/
…
RUN mkdir -p /naver/BASE-TFS/model
…
# Starting TFS.
COPY docker-entrypoint.sh /
RUN chmod 755 /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
APP_NAME=“aisp-app”
USERNAME=“deview”
MODEL_PATH=“/user/…”
AISP_TFS_MODEL_NAME=“sample_model”
AISP_TFS_OPTIONS=“—enable_batching=true
HOSTPORT=8080

13. •
•
# Install hdfscli for downloading model files.
RUN pip install hdfs
COPY files/.hdfscli.cfg /root/
…
RUN mkdir -p /naver/BASE-TFS/model
…
# Starting TFS.
COPY docker-entrypoint.sh /
RUN chmod 755 /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
# Upgrade c++ library.
RUN apt-get install -y software-properties-common
RUN add-apt-repository ppa:ubuntu-toolchain-r/test –y
RUN apt-get updateRUN apt-get upgrade -y libstdc++6
# Run following command if required version not installed
# strings /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | grep GLIBCXX

14. $ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
aisp/tfs-cpu 0.3 9370b0496efb 3 months ago 754.8 MB
docker.io/ubuntu 16.04 0458a4468cbc 7 months ago 111.7 MB
$ ./start_tfs_container.sh cpu # 안에 모델, 포트 등을 정의해놓음.
Docker running deview-tfs TFS container(CPU) ...
ad32740677d0203578f4eef8e3be132f3e73206659d9b30617e04519d77ff62d
$ docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
ad32740677d0 aisp/tfs-cpu:0.3 "/docker-entrypoint.s" 8 seconds ago Up 7 seconds 0.0.0.0:13377->80/tcp deview-tfs

16. •
•
•
•

17. •
•
•

18. •
•
•
•
•

19. •
•
•
•

20. •
•
•
•
•

21. •
•
•
•
auto length = tensor.dim_size(1); // Assuming 2D tensor with dim_size(0) = 1
auto data = reinterpret_cast<const float*>(tensor.tensor_data().data());
// 이제 data[i]로 접근

22. •
•
•
•

23. •
•
•
X
O

25. •
•
•
•
•
•

26. •
•
•

27. •
•

29. $ docker images # 1. 이미지 준비
REPOSITORY TAG IMAGE ID CREATED SIZE
aisp/aps 0.7 d1874e8f5559 3 months ago 1.623 GB
aisp/tfs-cpu 0.3 9370b0496efb 3 months ago 754.8 MB
$ cat start_tfs.sh # 2. 스크립트 설정
APP_NAME="deview-tfs"
HDFS_MODEL_PATH="/user/deview/session/model/20180920/05"
AISP_TFS_MODEL_NAME="samplemodel"
AISP_TFS_OPTIONS="--enable_batching=true"
HOSTPORT=13377
...
$ ./start_tfs.sh # 3. 실행
...
$ docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
e0ab6257c977 aisp/tfs-cpu:0.3 "/docker-entrypoint.s" 3 months ago Up 3 months 0.0.0.0:13377->80/tcp ncmp-tfs

30. # (dev container)
# ls
LinguistClient.cpp Makefile PredictionHelper.hpp RedisClient.hpp TfsClient.hpp deview_aps.cpp
deview_client.cpp utf8.h
LinguistClient.hpp PredictionHelper.cpp RedisClient.cpp TfsClient.cpp initRedis.cpp utf8
# make client
# ./deview_client 원피스
Request inference ...
output1
0.000169681 5.64217e-06 1.49844e-06 0.000512549 0.000931895 1.06733e-05 1.73642e-05 0.000400749 0.997623
0.00032458 2.61629e-06
output2
6.86965e-09 9.56301 ...

34. •
•

37. •
•
•

39. •

40. •
•
•
•
•
•
•
•
•