Skip to content

Commit 162542e

Browse files
GermeyGermey
committed
update docker compose
1 parent d4fca11 commit 162542e

9 files changed

Lines changed: 199 additions & 8 deletions

File tree

‎.dockerignore‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# Created by .ignore support plugin (hsz.mobi)
2+
### Python template
3+
# Byte-compiled / optimized / DLL files
4+
__pycache__/
5+
*.py[cod]
6+
*$py.class
7+
8+
# C extensions
9+
*.so
10+
11+
# Distribution / packaging
12+
.Python
13+
build/
14+
develop-eggs/
15+
dist/
16+
downloads/
17+
eggs/
18+
.eggs/
19+
lib/
20+
lib64/
21+
parts/
22+
sdist/
23+
var/
24+
wheels/
25+
pip-wheel-metadata/
26+
share/python-wheels/
27+
*.egg-info/
28+
.installed.cfg
29+
*.egg
30+
MANIFEST
31+
32+
# PyInstaller
33+
# Usually these files are written by a python script from a template
34+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
35+
*.manifest
36+
*.spec
37+
38+
# Installer logs
39+
pip-log.txt
40+
pip-delete-this-directory.txt
41+
42+
# Unit test / coverage reports
43+
htmlcov/
44+
.tox/
45+
.nox/
46+
.coverage
47+
.coverage.*
48+
.cache
49+
nosetests.xml
50+
coverage.xml
51+
*.cover
52+
*.py,cover
53+
.hypothesis/
54+
.pytest_cache/
55+
56+
# Translations
57+
*.mo
58+
*.pot
59+
60+
# Django stuff:
61+
*.log
62+
local_settings.py
63+
db.sqlite3
64+
db.sqlite3-journal
65+
66+
# Flask stuff:
67+
instance/
68+
.webassets-cache
69+
70+
# Scrapy stuff:
71+
.scrapy
72+
73+
# Sphinx documentation
74+
docs/_build/
75+
76+
# PyBuilder
77+
target/
78+
79+
# Jupyter Notebook
80+
.ipynb_checkpoints
81+
82+
# IPython
83+
profile_default/
84+
ipython_config.py
85+
86+
# pyenv
87+
.python-version
88+
89+
# pipenv
90+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
91+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
92+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
93+
# install all needed dependencies.
94+
#Pipfile.lock
95+
96+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
97+
__pypackages__/
98+
99+
# Celery stuff
100+
celerybeat-schedule
101+
celerybeat.pid
102+
103+
# SageMath parsed files
104+
*.sage.py
105+
106+
# Environments
107+
.env
108+
.venv
109+
env/
110+
venv/
111+
ENV/
112+
env.bak/
113+
venv.bak/
114+
115+
# Spyder project settings
116+
.spyderproject
117+
.spyproject
118+
119+
# Rope project settings
120+
.ropeproject
121+
122+
# mkdocs documentation
123+
/site
124+
125+
# mypy
126+
.mypy_cache/
127+
.dmypy.json
128+
dmypy.json
129+
130+
# Pyre type checker
131+
.pyre/

‎Dockerfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM python:3.6
2+
WORKDIR /app
3+
COPY . .
4+
RUN pip install -r requirements.txt
5+
CMD ["supervisord", "-c", "supervisord.conf"]

‎README.md‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,20 @@
11
# ProxyPool
22

3+
![](https://img.shields.io/badge/python-3.6%2B-brightgreen)
4+
5+
Welcome to ProxyPool.
6+
37
## Requirements
48

5-
* Docker
9+
Below is a list of requirements for ProxyPool.
10+
11+
* Docker
12+
* Docker-Compose
613

714
or
815

916
* Python: >=3.6
1017
* Redis
11-
* Environment: Virtual Env
1218

1319
## Run with Docker
1420

@@ -70,7 +76,9 @@ python3 run.py --processor tester
7076
python3 run.py --processor server
7177
```
7278

73-
### Usage
79+
## Usage
7480

7581
After running the ProxyPool, you can visit
76-
[http://localhost:5555/random](http://localhost:5555/) to access random proxy.
82+
[http://localhost:5555/random](http://localhost:5555/) to access random proxy.
83+
84+
Just enjoy it.

‎docker-compose.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
version: '3'
2+
services:
3+
redis:
4+
image: redis:alpine
5+
container_name: redis
6+
command: redis-server
7+
ports:
8+
- "6379:6379"
9+
restart: always
10+
proxypool:
11+
build: .
12+
image: 'germey/proxypool'
13+
container_name: proxypool
14+
ports:
15+
- "5555:5555"
16+
restart: always
17+
environment:
18+
REDIS_HOST: redis

‎proxypool/scheduler.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ def run_tester(self, cycle=CYCLE_TESTER):
2323
"""
2424
run tester
2525
"""
26+
if not ENABLE_TESTER:
27+
logger.info('tester not enabled, exit')
28+
return
2629
tester = Tester()
2730
loop = 0
2831
while True:
@@ -35,6 +38,9 @@ def run_getter(self, cycle=CYCLE_GETTER):
3538
"""
3639
run getter
3740
"""
41+
if not ENABLE_GETTER:
42+
logger.info('getter not enabled, exit')
43+
return
3844
getter = Getter()
3945
loop = 0
4046
while True:
@@ -47,6 +53,9 @@ def run_server(self):
4753
"""
4854
run server for api
4955
"""
56+
if not ENABLE_SERVER:
57+
logger.info('server not enabled, exit')
58+
return
5059
app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
5160

5261
def run(self):

‎proxypool/setting.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
from environs import Env
44
from loguru import logger
55

6+
from proxypool.utils.parse import parse_redis_connection_string
7+
68

79
env = Env()
810
env.read_env()
@@ -29,10 +31,10 @@
2931
# redis password, if no password, set it to None
3032
REDIS_PASSWORD = env.str('REDIS_PASSWORD', None)
3133
# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port
32-
# REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None)
34+
REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None)
3335

34-
# if REDIS_CONNECTION_STRING:
35-
# REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING)
36+
if REDIS_CONNECTION_STRING:
37+
REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING)
3638

3739
# redis hash table key name
3840
REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal')

‎requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,5 @@ requests==2.22.0
77
loguru==0.3.2
88
pyquery==1.4.0
99
attr==0.3.1
10+
supervisor==4.1.0
11+
redis==2.10.6

‎run.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
parser = argparse.ArgumentParser(description='ProxyPool')
66
parser.add_argument('--processor', type=str, help='processor to run')
77
args = parser.parse_args()
8-
print('args', args)
98

109
if __name__ == '__main__':
1110
# if processor set, just run it

‎supervisord.conf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[supervisord]
2+
nodaemon=true
3+
4+
[program:tester]
5+
process_name=tester
6+
command=python3 run.py --processor tester
7+
directory=/app
8+
9+
[program:getter]
10+
process_name=getter
11+
command=python3 run.py --processor getter
12+
directory=/app
13+
14+
[program:server]
15+
process_name=server
16+
command=python3 run.py --processor server
17+
directory=/app

0 commit comments

Comments
 (0)