daocloud安装pyspider

信息技术
docker, pyspider
编辑 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
phantomjs:
  image: binux/pyspider:latest
  command: phantomjs
  cpu_shares: 512
  environment:
  - EXCLUDE_PORTS=5000,23333,24444
  expose:
  - '25555'
  mem_limit: 512m
  restart: always
phantomjs-lb:
  image: dockercloud/haproxy:latest
  links:
  - phantomjs
  restart: always
fetcher:
  image: binux/pyspider:latest
  links:
  - phantomjs-lb:phantomjs
  command: --message-queue "redis://pyredis:6379/1" --phantomjs-proxy "phantomjs:80"
    fetcher --xmlrpc
  cpu_shares: 512
  environment:
  - EXCLUDE_PORTS=5000,25555,23333
  external_links:
  - dao_redis_1:pyredis
  mem_limit: 128m
  restart: always
fetcher-lb:
  image: dockercloud/haproxy:latest
  links:
  - fetcher
  restart: always
processor:
  image: binux/pyspider:latest
  command: --projectdb "mysql+projectdb://root:deffyc126@dao_pymysql_1/projectdb"
    --message-queue "redis://pyredis:6379/1"
  cpu_shares: 512
  mem_limit: 256m
  external_links:
  - dao_pymysql_1
  - dao_redis_1:pyredis
  restart: always
result-worker:
  image: binux/pyspider:latest
  command: --taskdb "mysql+taskdb://root:deffyc126@dao_pymysql_1:3306/taskdb"  --projectdb
    "mysql+projectdb://root:deffyc126@dao_pymysql_1:3306/projectdb" --resultdb "mysql+resultdb://root:deffyc126@dao_pymysql_1:3306/resultdb"
    --message-queue "redis://pyredis:6379/1"
  cpu_shares: 512
  mem_limit: 256m
  external_links:
  - dao_pymysql_1
  - dao_redis_1:pyredis
  restart: always
scheduler:
  image: binux/pyspider:latest
  command: --taskdb "mysql+taskdb://root:deffyc126@dao_pymysql_1/taskdb" --resultdb
    "mysql+resultdb://root:deffyc126@dao_pymysql_1/resultdb" --projectdb "mysql+projectdb://root:deffyc126@dao_pymysql_1/projectdb"
    --message-queue "redis://pyredis:6379/1" scheduler --inqueue-limit 5000 --xmlrpc
    --delete-time 43200
  external_links:
  - dao_pymysql_1
  - dao_redis_1:pyredis
webui:
  image: binux/pyspider:latest
  command: --taskdb "mysql+taskdb://root:deffyc126@pymysql/taskdb"  --projectdb "mysql+projectdb://root:deffyc126@pymysql/projectdb"
    --resultdb "mysql+resultdb://root:deffyc126@pymysql/resultdb" --message-queue
    "redis://pyredis:6379/1" webui --max-rate 0.2 --max-burst 3 --scheduler-rpc "http://scheduler:23333/"
    --fetcher-rpc "http://fetcher/" --username "deffyc" --password "deffyc126"
    --need-auth
  cpu_shares: 512
  environment:
  - EXCLUDE_PORTS=24444,25555,23333
  links:
  - fetcher-lb:fetcher
  - scheduler
  external_links:
  - dao_pymysql_1:pymysql
  - dao_redis_1:pyredis
  mem_limit: 256m
  restart: always
webui-lb:
  image: dockercloud/haproxy:latest
  links:
  - webui
  restart: always
nginx:
  image: daocloud.io/library/nginx:stable
  links:
  - webui-lb:HAPROXY
  ports:
  - 0.0.0.0:80:80
  - 0.0.0.0:443:443
  volumes:
  - /root/profile/nginx/nginx.conf:/etc/nginx/nginx.conf
  - /root/profile/nginx/conf.d/:/etc/nginx/conf.d/
  restart: always

docker-compose scale phantomjs=2 processor=2 webui=4