From 8a3d65aebcfcc2510570093e69617e70848a2cb9 Mon Sep 17 00:00:00 2001 From: Michael Wills Date: Tue, 1 Mar 2016 15:27:41 -0800 Subject: [PATCH 01/55] Adding boto3 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b5595c8..dd944a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,4 @@ syncano==4.1.0 fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 +boto3==1.2.5 From 5abc29b5c2b300e0c26325eecef0d55142cd3b56 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Thu, 17 Mar 2016 16:23:59 +0100 Subject: [PATCH 02/55] [INFRA-251]: old syncano version in virtual env --- Dockerfile | 23 ++++++++++++++++------- README.md | 2 +- circle.yml | 8 ++++---- requirements.txt | 2 +- test2.py | 2 ++ test3.py | 2 ++ 6 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 test2.py create mode 100644 test3.py diff --git a/Dockerfile b/Dockerfile index cea5382..7a549ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:trusty MAINTAINER "Syncano DevOps Team" -ENV LAST_REFRESHED 2016-01-11 +ENV LAST_REFRESHED 2016-03-17 ENV export SYNCANO_APIROOT='https://api.syncano.io/' COPY requirements.txt /tmp/requirements.txt @@ -15,12 +15,21 @@ RUN apt-get update && apt-get install -qqy \ python-dev \ python-numpy \ python-scipy \ - wget \ - && wget https://bootstrap.pypa.io/get-pip.py \ - && python get-pip.py \ - && pip install --upgrade pip \ - && pip install -r /tmp/requirements.txt \ - && pip install -r /tmp/external_requirements.txt + wget && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py && \ + pip install --upgrade pip && \ + pip install -r /tmp/requirements.txt && \ + pip install -r /tmp/external_requirements.txt && \ + pip install syncano==4.2.0 + +RUN mkdir /home/syncano && \ + cd /home/syncano/ && \ + virtualenv deprecated && \ + . new_syncano/bin/activate && \ + pip install -r /tmp/requirements.txt && \ + pip install -r /tmp/external_requirements.txt && \ + pip install syncano==4.1.0 # create a special user to run code # user without root privileges greatly improves security diff --git a/README.md b/README.md index 5263e3d..9d6f9c1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Type "help", "copyright", "credits" or "license" for more information. >>> ``` -In a container, you can use Syncano's Python Library: +In a container you can use Syncano's Python Library: ``` >>> import syncano diff --git a/circle.yml b/circle.yml index 2e9e5fd..d74cb23 100644 --- a/circle.yml +++ b/circle.yml @@ -4,12 +4,14 @@ machine: dependencies: override: - - sed -i 's/CMD/#CMD/g' Dockerfile - docker build -t quay.io/syncano/python-codebox . test: override: - - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/deprecated/bin/python /tmp/test.py + - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/deprecated/bin/python /tmp/test.py + - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py deployment: production: @@ -17,6 +19,4 @@ deployment: - master commands: - sed -e "s||$DOCKER_REGISTRY|g" -e "s||$DOCKER_EMAIL|g" -e "s||$DOCKER_AUTH|g" < .dockercfg.template > ~/.dockercfg - - sed -i 's/#CMD/CMD/g' Dockerfile - - docker build -t quay.io/syncano/python-codebox . - docker push quay.io/syncano/python-codebox diff --git a/requirements.txt b/requirements.txt index dd944a3..3483752 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,8 +19,8 @@ requests-oauthlib==0.5.0 tweepy==3.4.0 braintree==3.20.0 googleads==3.11.0 -syncano==4.1.0 fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 +virtualenv==13.0.3 diff --git a/test2.py b/test2.py new file mode 100644 index 0000000..a8ff144 --- /dev/null +++ b/test2.py @@ -0,0 +1,2 @@ +import syncano +assert syncano.__version__ == '4.1.0' diff --git a/test3.py b/test3.py new file mode 100644 index 0000000..a1e37cf --- /dev/null +++ b/test3.py @@ -0,0 +1,2 @@ +import syncano +assert syncano.__version__ == '4.2.0' From d1910617fa86e26c19bf023f3b7b85c465804a4b Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Thu, 17 Mar 2016 16:35:32 +0100 Subject: [PATCH 03/55] [INFRA-251]: corrected; moved new version to env --- Dockerfile | 8 ++++---- circle.yml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7a549ee..0259ccc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,15 +21,15 @@ RUN apt-get update && apt-get install -qqy \ pip install --upgrade pip && \ pip install -r /tmp/requirements.txt && \ pip install -r /tmp/external_requirements.txt && \ - pip install syncano==4.2.0 + pip install syncano==4.1.0 RUN mkdir /home/syncano && \ cd /home/syncano/ && \ - virtualenv deprecated && \ - . new_syncano/bin/activate && \ + virtualenv new && \ + . new/bin/activate && \ pip install -r /tmp/requirements.txt && \ pip install -r /tmp/external_requirements.txt && \ - pip install syncano==4.1.0 + pip install syncano==4.2.0 # create a special user to run code # user without root privileges greatly improves security diff --git a/circle.yml b/circle.yml index d74cb23..51979dd 100644 --- a/circle.yml +++ b/circle.yml @@ -9,9 +9,9 @@ dependencies: test: override: - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/deprecated/bin/python /tmp/test.py - - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/deprecated/bin/python /tmp/test.py - - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/new/bin/python /tmp/test.py + - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py + - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/new/bin/python /tmp/test.py deployment: production: From e49c97436e2126a7f49b6fc442100fed45ebb23e Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 18 Mar 2016 11:55:14 +0100 Subject: [PATCH 04/55] [INFRA-251]: separate requirements and envs --- Dockerfile | 31 ++++++++++++++++++------------- circle.yml | 4 ++-- requirements.txt | 2 +- requirements_base.txt | 1 + requirements_v41.txt | 26 ++++++++++++++++++++++++++ 5 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 requirements_base.txt create mode 100644 requirements_v41.txt diff --git a/Dockerfile b/Dockerfile index 0259ccc..b8e263a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,10 @@ FROM ubuntu:trusty MAINTAINER "Syncano DevOps Team" -ENV LAST_REFRESHED 2016-03-17 +ENV LAST_REFRESHED 2016-03-18 ENV export SYNCANO_APIROOT='https://api.syncano.io/' -COPY requirements.txt /tmp/requirements.txt -COPY external_requirements.txt /tmp/external_requirements.txt +COPY *requirements*.txt /tmp/ RUN apt-get update && apt-get install -qqy \ git \ @@ -19,22 +18,28 @@ RUN apt-get update && apt-get install -qqy \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py && \ pip install --upgrade pip && \ - pip install -r /tmp/requirements.txt && \ - pip install -r /tmp/external_requirements.txt && \ - pip install syncano==4.1.0 + pip install -r /tmp/requirements_base.txt && \ + mkdir /home/syncano + +RUN cd /home/syncano/ && \ + virtualenv v4.1 && \ + . v4.1/bin/activate && \ + pip install -r /tmp/requirements_v41.txt && \ + pip install -r /tmp/external_requirements.txt -RUN mkdir /home/syncano && \ - cd /home/syncano/ && \ - virtualenv new && \ - . new/bin/activate && \ +RUN cd /home/syncano/ && \ + virtualenv v4.2 && \ + . v4.2/bin/activate && \ pip install -r /tmp/requirements.txt && \ - pip install -r /tmp/external_requirements.txt && \ - pip install syncano==4.2.0 + pip install -r /tmp/external_requirements.txt +RUN ln -sf /home/syncano/v4.1/bin/python /usr/bin/python && \ + ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-42 && \ + ln -sf /home/syncano/v4.1/bin/python /usr/bin/python27-41 # create a special user to run code # user without root privileges greatly improves security RUN useradd syncano -d /tmp -s /bin/bash RUN chmod 1777 /tmp USER syncano -CMD ["python"] +CMD "python" diff --git a/circle.yml b/circle.yml index 51979dd..85989ad 100644 --- a/circle.yml +++ b/circle.yml @@ -9,9 +9,9 @@ dependencies: test: override: - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/new/bin/python /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-42 /tmp/test.py - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox /home/syncano/new/bin/python /tmp/test.py + - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python27-42 /tmp/test.py deployment: production: diff --git a/requirements.txt b/requirements.txt index 3483752..b711e79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,4 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -virtualenv==13.0.3 +syncano==4.2.0 diff --git a/requirements_base.txt b/requirements_base.txt new file mode 100644 index 0000000..0f3f52d --- /dev/null +++ b/requirements_base.txt @@ -0,0 +1 @@ +virtualenv==13.0.3 diff --git a/requirements_v41.txt b/requirements_v41.txt new file mode 100644 index 0000000..4f0432d --- /dev/null +++ b/requirements_v41.txt @@ -0,0 +1,26 @@ +Pillow==2.7.0 +requests==2.7.0 +icalendar==3.8.4 +pytz==2014.10 +html2text==2015.2.18 +pkgtools==0.7.3 +names==0.3.0 +httplib2==0.9.1 +twilio==4.4.0 +qrcode==5.1 +analytics-python==1.0.3 +osa==0.1.6.6 +google-api-python-client==1.4.1 +six==1.9.0 +uritemplate==0.6 +oauth2client==1.4.11 +oauthlib==1.0.3 +requests-oauthlib==0.5.0 +tweepy==3.4.0 +braintree==3.20.0 +googleads==3.11.0 +fastly==0.0.2 +pandas==0.17.1 +BeautifulSoup==3.2.1 +boto3==1.2.5 +syncano==4.1.0 From c992451b3f4e056d48c96e07f35e1b7467c4f469 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 18 Mar 2016 13:01:59 +0100 Subject: [PATCH 05/55] [INFRA-251]: use workdir instead of cd [ci skip] --- Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index b8e263a..4f6529f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,14 +21,14 @@ RUN apt-get update && apt-get install -qqy \ pip install -r /tmp/requirements_base.txt && \ mkdir /home/syncano -RUN cd /home/syncano/ && \ - virtualenv v4.1 && \ +WORKDIR /home/syncano/ +RUN virtualenv v4.1 && \ . v4.1/bin/activate && \ pip install -r /tmp/requirements_v41.txt && \ - pip install -r /tmp/external_requirements.txt + pip install -r /tmp/external_requirements.txt && \ + deactivate -RUN cd /home/syncano/ && \ - virtualenv v4.2 && \ +RUN virtualenv v4.2 && \ . v4.2/bin/activate && \ pip install -r /tmp/requirements.txt && \ pip install -r /tmp/external_requirements.txt @@ -41,5 +41,6 @@ RUN ln -sf /home/syncano/v4.1/bin/python /usr/bin/python && \ RUN useradd syncano -d /tmp -s /bin/bash RUN chmod 1777 /tmp +WORKDIR /tmp USER syncano CMD "python" From 3503255e624ecc01dbfe920b327085c75880c6e6 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Mon, 21 Mar 2016 13:03:23 +0100 Subject: [PATCH 06/55] [INFRA-251]: remove pip upgrade --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4f6529f..302f09c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,6 @@ RUN apt-get update && apt-get install -qqy \ wget && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py && \ - pip install --upgrade pip && \ pip install -r /tmp/requirements_base.txt && \ mkdir /home/syncano From bd7a4718604564745692a3e15c2f5c45c2398f4a Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 23 Mar 2016 10:32:40 +0100 Subject: [PATCH 07/55] [INFRA-260]: new syncano python lib release --- Dockerfile | 16 ++++++++-------- circle.yml | 4 ++-- requirements.txt | 2 +- requirements_v41.txt => requirements_v42.txt | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) rename requirements_v41.txt => requirements_v42.txt (96%) diff --git a/Dockerfile b/Dockerfile index 302f09c..f1872cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,20 +21,20 @@ RUN apt-get update && apt-get install -qqy \ mkdir /home/syncano WORKDIR /home/syncano/ -RUN virtualenv v4.1 && \ - . v4.1/bin/activate && \ - pip install -r /tmp/requirements_v41.txt && \ +RUN virtualenv v4.2 && \ + . v4.2/bin/activate && \ + pip install -r /tmp/requirements_v42.txt && \ pip install -r /tmp/external_requirements.txt && \ deactivate -RUN virtualenv v4.2 && \ - . v4.2/bin/activate && \ +RUN virtualenv v5.0 && \ + . v5.0/bin/activate && \ pip install -r /tmp/requirements.txt && \ pip install -r /tmp/external_requirements.txt -RUN ln -sf /home/syncano/v4.1/bin/python /usr/bin/python && \ - ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-42 && \ - ln -sf /home/syncano/v4.1/bin/python /usr/bin/python27-41 +RUN ln -sf /home/syncano/v4.2/bin/python /usr/bin/python && \ + ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-50 && \ + ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-42 # create a special user to run code # user without root privileges greatly improves security RUN useradd syncano -d /tmp -s /bin/bash diff --git a/circle.yml b/circle.yml index 85989ad..2662882 100644 --- a/circle.yml +++ b/circle.yml @@ -9,9 +9,9 @@ dependencies: test: override: - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-42 /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-50 /tmp/test.py - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python27-42 /tmp/test.py + - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python27-50 /tmp/test.py deployment: production: diff --git a/requirements.txt b/requirements.txt index b711e79..7561291 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,4 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==4.2.0 +git+https://github.com/Syncano/syncano-python.git@develop diff --git a/requirements_v41.txt b/requirements_v42.txt similarity index 96% rename from requirements_v41.txt rename to requirements_v42.txt index 4f0432d..b711e79 100644 --- a/requirements_v41.txt +++ b/requirements_v42.txt @@ -23,4 +23,4 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==4.1.0 +syncano==4.2.0 From 4d23b835493d01f7bf8aa81843931129640f6200 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 23 Mar 2016 10:38:53 +0100 Subject: [PATCH 08/55] [INFRA-260]: correct symlinks and version testing --- Dockerfile | 4 ++-- circle.yml | 6 ++++-- test2.py | 2 +- test3.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1872cc..527849b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,8 +33,8 @@ RUN virtualenv v5.0 && \ pip install -r /tmp/external_requirements.txt RUN ln -sf /home/syncano/v4.2/bin/python /usr/bin/python && \ - ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-50 && \ - ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-42 + ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-lib5.0 && \ + ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 # create a special user to run code # user without root privileges greatly improves security RUN useradd syncano -d /tmp -s /bin/bash diff --git a/circle.yml b/circle.yml index 2662882..f947919 100644 --- a/circle.yml +++ b/circle.yml @@ -9,9 +9,11 @@ dependencies: test: override: - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-50 /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python27-50 /tmp/test.py + - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py + - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py deployment: production: diff --git a/test2.py b/test2.py index a8ff144..a1e37cf 100644 --- a/test2.py +++ b/test2.py @@ -1,2 +1,2 @@ import syncano -assert syncano.__version__ == '4.1.0' +assert syncano.__version__ == '4.2.0' diff --git a/test3.py b/test3.py index a1e37cf..43d5f1f 100644 --- a/test3.py +++ b/test3.py @@ -1,2 +1,2 @@ import syncano -assert syncano.__version__ == '4.2.0' +assert syncano.__version__ == '5.0.0' From b407cadc7b56891dd0cdce8cce1a160a7f8112fa Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 23 Mar 2016 10:49:32 +0100 Subject: [PATCH 09/55] [INFRA-260]: syncano version bump --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7561291..3b477a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,4 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -git+https://github.com/Syncano/syncano-python.git@develop +syncano==5.0.0 From afdf437f476ef0f8dba997ee40bf1c45efc343fe Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Wed, 23 Mar 2016 12:38:02 +0100 Subject: [PATCH 10/55] [default_version] Newer version as default --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 527849b..b9640f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ RUN virtualenv v5.0 && \ pip install -r /tmp/requirements.txt && \ pip install -r /tmp/external_requirements.txt -RUN ln -sf /home/syncano/v4.2/bin/python /usr/bin/python && \ +RUN ln -sf /home/syncano/v5.0/bin/python /usr/bin/python && \ ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-lib5.0 && \ ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 # create a special user to run code From 6a50516f8241e37ccbb3839086f4410ebe8ca6b1 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 25 Mar 2016 01:06:33 +0100 Subject: [PATCH 11/55] [INFRA-260]: download envs from s3 --- Dockerfile | 7 ++++--- circle.yml | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 527849b..5b7149b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ FROM ubuntu:trusty MAINTAINER "Syncano DevOps Team" -ENV LAST_REFRESHED 2016-03-18 +ENV LAST_REFRESHED 2016-03-25 ENV export SYNCANO_APIROOT='https://api.syncano.io/' COPY *requirements*.txt /tmp/ +COPY *.tar.gz /tmp/ RUN apt-get update && apt-get install -qqy \ git \ @@ -21,13 +22,13 @@ RUN apt-get update && apt-get install -qqy \ mkdir /home/syncano WORKDIR /home/syncano/ -RUN virtualenv v4.2 && \ +RUN tar xzvf /tmp/42.tar.gz && \ . v4.2/bin/activate && \ pip install -r /tmp/requirements_v42.txt && \ pip install -r /tmp/external_requirements.txt && \ deactivate -RUN virtualenv v5.0 && \ +RUN tar xzvf /tmp/50.tar.gz && \ . v5.0/bin/activate && \ pip install -r /tmp/requirements.txt && \ pip install -r /tmp/external_requirements.txt diff --git a/circle.yml b/circle.yml index f947919..4857807 100644 --- a/circle.yml +++ b/circle.yml @@ -4,6 +4,8 @@ machine: dependencies: override: + - wget https://s3.amazonaws.com/codebox-bucket/python/42.tar.gz + - wget https://s3.amazonaws.com/codebox-bucket/python/50.tar.gz - docker build -t quay.io/syncano/python-codebox . test: From e7e0e33bc184830199249418e5231b7e91773398 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 25 Mar 2016 08:44:55 +0100 Subject: [PATCH 12/55] [INFRA-251]: commont syncano user creation --- Dockerfile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b7149b..87975c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,16 @@ FROM ubuntu:trusty MAINTAINER "Syncano DevOps Team" ENV LAST_REFRESHED 2016-03-25 -ENV export SYNCANO_APIROOT='https://api.syncano.io/' +ENV SYNCANO_APIROOT https://api.syncano.io/ + +RUN groupadd -r syncano && \ + useradd -u 1000 -r -g syncano syncano -d /tmp -s /bin/bash && \ + mkdir /home/syncano && \ + chown -R syncano /home/syncano + +# enable everyone to use /tmp +RUN chmod 1777 /tmp +# -- CUT -- COPY *requirements*.txt /tmp/ COPY *.tar.gz /tmp/ @@ -36,10 +45,6 @@ RUN tar xzvf /tmp/50.tar.gz && \ RUN ln -sf /home/syncano/v4.2/bin/python /usr/bin/python && \ ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-lib5.0 && \ ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 -# create a special user to run code -# user without root privileges greatly improves security -RUN useradd syncano -d /tmp -s /bin/bash -RUN chmod 1777 /tmp WORKDIR /tmp USER syncano From 22ee03e4642b881ba4ec7e0961d5b7b80b5f5b6f Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 25 Mar 2016 08:59:39 +0100 Subject: [PATCH 13/55] [INFRA-251]: remove redundant mkdir --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 87975c4..820aa81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,8 +27,7 @@ RUN apt-get update && apt-get install -qqy \ wget && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py && \ - pip install -r /tmp/requirements_base.txt && \ - mkdir /home/syncano + pip install -r /tmp/requirements_base.txt WORKDIR /home/syncano/ RUN tar xzvf /tmp/42.tar.gz && \ From 345a6c39ade7b294b38f0f7ce7ac222e3486c394 Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Fri, 25 Mar 2016 11:17:08 +0100 Subject: [PATCH 14/55] [default_version] cleanup and test fix --- circle.yml | 6 +++--- test2.py => test_v42.py | 0 test3.py => test_v50.py | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename test2.py => test_v42.py (100%) rename test3.py => test_v50.py (100%) diff --git a/circle.yml b/circle.yml index f947919..8895939 100644 --- a/circle.yml +++ b/circle.yml @@ -11,9 +11,9 @@ test: - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py - - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - - docker run -it -v `pwd`/test2.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py - - docker run -it -v `pwd`/test3.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py + - docker run -it -v `pwd`/test_v50.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py + - docker run -it -v `pwd`/test_v42.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py + - docker run -it -v `pwd`/test_v50.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py deployment: production: diff --git a/test2.py b/test_v42.py similarity index 100% rename from test2.py rename to test_v42.py diff --git a/test3.py b/test_v50.py similarity index 100% rename from test3.py rename to test_v50.py From b9325c4c3aa58ee709792ad8c135cc68d9376981 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 25 Mar 2016 11:42:39 +0100 Subject: [PATCH 15/55] [INFRA-251]: cut being, cut end [ci skip] --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 820aa81..f2200b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN groupadd -r syncano && \ # enable everyone to use /tmp RUN chmod 1777 /tmp -# -- CUT -- +# -- CUT BEGIN -- COPY *requirements*.txt /tmp/ COPY *.tar.gz /tmp/ @@ -45,6 +45,7 @@ RUN ln -sf /home/syncano/v4.2/bin/python /usr/bin/python && \ ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-lib5.0 && \ ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 -WORKDIR /tmp +# -- CUT END -- USER syncano +WORKDIR /tmp CMD "python" From ffd58c646e0ae39884cc5ccaeb95014c18e49842 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Tue, 29 Mar 2016 22:11:00 +0200 Subject: [PATCH 16/55] [CORE-1412]: fix scipy installation --- Dockerfile | 6 ++++-- test_v42.py | 4 ++-- test_v50.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1f770f..ba68305 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,13 +30,15 @@ RUN apt-get update && apt-get install -qqy \ pip install -r /tmp/requirements_base.txt WORKDIR /home/syncano/ -RUN tar xzvf /tmp/42.tar.gz && \ +RUN virtualenv --system-site-packages v4.2 && \ + tar xzvf /tmp/42.tar.gz --skip-old-files && \ . v4.2/bin/activate && \ pip install -r /tmp/requirements_v42.txt && \ pip install -r /tmp/external_requirements.txt && \ deactivate -RUN tar xzvf /tmp/50.tar.gz && \ +RUN virtualenv --system-site-packages v5.0 && \ + tar xzvf /tmp/50.tar.gz --skip-old-files && \ . v5.0/bin/activate && \ pip install -r /tmp/requirements.txt && \ pip install -r /tmp/external_requirements.txt diff --git a/test_v42.py b/test_v42.py index a1e37cf..64d4aad 100644 --- a/test_v42.py +++ b/test_v42.py @@ -1,2 +1,2 @@ -import syncano -assert syncano.__version__ == '4.2.0' +for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '4.2.0')): + assert __import__(lib).__version__ == version diff --git a/test_v50.py b/test_v50.py index 43d5f1f..552e7c9 100644 --- a/test_v50.py +++ b/test_v50.py @@ -1,2 +1,2 @@ -import syncano -assert syncano.__version__ == '5.0.0' +for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.0.0')): + assert __import__(lib).__version__ == version From f3719669ea5c384c2b1514706d6dc5a0d088b1da Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Mon, 4 Apr 2016 18:48:44 +0200 Subject: [PATCH 17/55] Add mixpanel --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3b477a8..6635d24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 syncano==5.0.0 +mixpanel==4.3.0 From 637c085145715d068565b19ec018c9e4478e6cf2 Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Mon, 4 Apr 2016 18:49:22 +0200 Subject: [PATCH 18/55] Add mixpanel to v4.2 --- requirements_v42.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_v42.txt b/requirements_v42.txt index b711e79..4d31ddc 100644 --- a/requirements_v42.txt +++ b/requirements_v42.txt @@ -24,3 +24,4 @@ pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 syncano==4.2.0 +mixpanel==4.3.0 From 60dd13185638e9e0bfd9a7230d2b76fdff6578fc Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Mon, 4 Apr 2016 20:40:59 +0200 Subject: [PATCH 19/55] [ci skip] Add mixpanel query --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6635d24..01f42d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ BeautifulSoup==3.2.1 boto3==1.2.5 syncano==5.0.0 mixpanel==4.3.0 +mixpanel-query-py==0.1.7 From 7891778b58379e5ea8dc4830ad872d079831db3d Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Mon, 4 Apr 2016 20:41:22 +0200 Subject: [PATCH 20/55] Add mixpanel query to v4.2 --- requirements_v42.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_v42.txt b/requirements_v42.txt index 4d31ddc..89009e8 100644 --- a/requirements_v42.txt +++ b/requirements_v42.txt @@ -25,3 +25,4 @@ BeautifulSoup==3.2.1 boto3==1.2.5 syncano==4.2.0 mixpanel==4.3.0 +mixpanel-query-py==0.1.7 From bb299cce817a356eb17822c80dfd2806380c8834 Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Wed, 6 Apr 2016 15:26:07 +0200 Subject: [PATCH 21/55] [ci skip] Remove mixpanel from v4.2 --- requirements_v42.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements_v42.txt b/requirements_v42.txt index 89009e8..b711e79 100644 --- a/requirements_v42.txt +++ b/requirements_v42.txt @@ -24,5 +24,3 @@ pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 syncano==4.2.0 -mixpanel==4.3.0 -mixpanel-query-py==0.1.7 From 2990e6c7c1cbc3b32cc0746960a8f0719effb7aa Mon Sep 17 00:00:00 2001 From: Daniel Kopka Date: Thu, 7 Apr 2016 11:24:48 +0200 Subject: [PATCH 22/55] Python lib bump to v5.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 01f42d8..88fafd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,6 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==5.0.0 +syncano==5.0.1 mixpanel==4.3.0 mixpanel-query-py==0.1.7 From 64d43c36f18ca9d10af3cf4f0556d1e82c7b55e4 Mon Sep 17 00:00:00 2001 From: Daniel Kopka Date: Thu, 7 Apr 2016 11:47:51 +0200 Subject: [PATCH 23/55] assert fix --- test_v50.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_v50.py b/test_v50.py index 552e7c9..dbc005b 100644 --- a/test_v50.py +++ b/test_v50.py @@ -1,2 +1,2 @@ -for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.0.0')): +for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.0.1')): assert __import__(lib).__version__ == version From 81a18ede4fe231f4fd4c5d9f36d136fe8400d254 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 22 Apr 2016 16:36:07 +0200 Subject: [PATCH 24/55] [INFRA-235]: add script refresh [INFRA-235]: PR comments --- circle.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/circle.yml b/circle.yml index 1fab223..4e170ab 100644 --- a/circle.yml +++ b/circle.yml @@ -22,5 +22,6 @@ deployment: branch: - master commands: + - curl -X POST $REFRESH_SCRIPT_URL - sed -e "s||$DOCKER_REGISTRY|g" -e "s||$DOCKER_EMAIL|g" -e "s||$DOCKER_AUTH|g" < .dockercfg.template > ~/.dockercfg - docker push quay.io/syncano/python-codebox From 051ea0f1db809b0800cdd4119612ec3e708afbcc Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 11 May 2016 15:14:30 +0200 Subject: [PATCH 25/55] [CORE-1347]: python3 enabled [CORE-1347]: numpy and scipy for python3 [CORE-1347]: add missing file [CORE-1347]: correct python3 numpy installation [CORE-1347]: remove unused file [CORE-1347]: add python3 reqs --- Dockerfile | 13 ++++++++++++- circle.yml | 5 +++++ requirements_python3.txt | 28 ++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 requirements_python3.txt diff --git a/Dockerfile b/Dockerfile index ba68305..b4680c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,11 @@ RUN apt-get update && apt-get install -qqy \ libssl-dev \ libjpeg-dev \ python-dev \ + python3-dev \ python-numpy \ + python3-numpy \ python-scipy \ + python3-scipy \ wget && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py && \ @@ -41,11 +44,19 @@ RUN virtualenv --system-site-packages v5.0 && \ tar xzvf /tmp/50.tar.gz --skip-old-files && \ . v5.0/bin/activate && \ pip install -r /tmp/requirements.txt && \ + pip install -r /tmp/external_requirements.txt && \ + deactivate + +RUN virtualenv --system-site-packages -p python3 p3v5.0 && \ + . p3v5.0/bin/activate && \ + pip install -r /tmp/requirements_python3.txt && \ pip install -r /tmp/external_requirements.txt RUN ln -sf /home/syncano/v5.0/bin/python /usr/bin/python && \ ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-lib5.0 && \ - ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 + ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 && \ + ln -sf /home/syncano/p3v5.0/bin/python /usr/bin/python3 && \ + ln -sf /home/syncano/p3v5.0/bin/python /usr/bin/python3-lib5.0 # -- CUT END -- USER syncano diff --git a/circle.yml b/circle.yml index 4e170ab..3546aec 100644 --- a/circle.yml +++ b/circle.yml @@ -12,11 +12,16 @@ test: override: - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py - docker run -it -v `pwd`/test_v50.py:/tmp/test.py quay.io/syncano/python-codebox python /tmp/test.py + - docker run -it -v `pwd`/test_v42.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py - docker run -it -v `pwd`/test_v50.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python3 /tmp/test.py + - docker run -it -v `pwd`/test_v50.py:/tmp/test.py quay.io/syncano/python-codebox python3-lib5.0 /tmp/test.py + deployment: production: branch: diff --git a/requirements_python3.txt b/requirements_python3.txt new file mode 100644 index 0000000..847316c --- /dev/null +++ b/requirements_python3.txt @@ -0,0 +1,28 @@ +Pillow==2.7.0 +requests==2.7.0 +icalendar==3.8.4 +pytz==2014.10 +html2text==2015.2.18 +pkgtools==0.7.3 +names==0.3.0 +httplib2==0.9.1 +twilio==4.4.0 +qrcode==5.1 +analytics-python==1.0.3 +osa==0.1.6.6 +google-api-python-client==1.4.1 +six==1.9.0 +uritemplate==0.6 +oauth2client==1.4.11 +oauthlib==1.0.3 +requests-oauthlib==0.5.0 +tweepy==3.4.0 +braintree==3.20.0 +googleads==3.11.0 +fastly==0.0.2 +pandas==0.17.1 +beautifulsoup4==4.4.1 +boto3==1.2.5 +syncano==5.0.1 +mixpanel==4.3.0 +mixpanel-query-py==0.1.7 From b358141565b71a8fcb931429278b0d6f6def449c Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Thu, 12 May 2016 10:30:08 +0200 Subject: [PATCH 26/55] [CORE-1347]: add python3 version check --- circle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/circle.yml b/circle.yml index 3546aec..2ea99f5 100644 --- a/circle.yml +++ b/circle.yml @@ -19,6 +19,8 @@ test: - docker run -it -v `pwd`/test_v42.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib4.2 /tmp/test.py - docker run -it -v `pwd`/test_v50.py:/tmp/test.py quay.io/syncano/python-codebox python27-lib5.0 /tmp/test.py + - if docker run -it quay.io/syncano/python-codebox python3 --version | grep -vq "Python 3.4"; then exit 1; fi + - docker run -it -v `pwd`/test.py:/tmp/test.py quay.io/syncano/python-codebox python3 /tmp/test.py - docker run -it -v `pwd`/test_v50.py:/tmp/test.py quay.io/syncano/python-codebox python3-lib5.0 /tmp/test.py From 1d82800859e363e91b77c757a6724a2214b20055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 24 May 2016 10:57:50 +0200 Subject: [PATCH 27/55] [CORE-1518] update the syncano LIB version; --- requirements_python3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_python3.txt b/requirements_python3.txt index 847316c..b33554d 100644 --- a/requirements_python3.txt +++ b/requirements_python3.txt @@ -23,6 +23,6 @@ fastly==0.0.2 pandas==0.17.1 beautifulsoup4==4.4.1 boto3==1.2.5 -syncano==5.0.1 +syncano==5.1.0 mixpanel==4.3.0 mixpanel-query-py==0.1.7 From 9b250ff621736c696df4f25ec918e0001398954a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 24 May 2016 11:00:09 +0200 Subject: [PATCH 28/55] [CORE-1518] update the test with new LIB version; --- test_v50.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_v50.py b/test_v50.py index dbc005b..445efb5 100644 --- a/test_v50.py +++ b/test_v50.py @@ -1,2 +1,2 @@ -for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.0.1')): +for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.1.0')): assert __import__(lib).__version__ == version From caf148fb03b57e515ee4e8edd9138090cbd26caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 24 May 2016 11:17:03 +0200 Subject: [PATCH 29/55] [CORE-1518] update one more req file :) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 88fafd2..7f851df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,6 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==5.0.1 +syncano==5.1.0 mixpanel==4.3.0 mixpanel-query-py==0.1.7 From 1ec8bf8485a46c781bc51bdc7a1482a8ab1f461a Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 25 May 2016 10:58:57 +0200 Subject: [PATCH 30/55] [INFRA-294]: lib version fix --- external_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external_requirements.txt b/external_requirements.txt index 4bee3b1..c5a4b74 100644 --- a/external_requirements.txt +++ b/external_requirements.txt @@ -1,3 +1,3 @@ -i https://pypi.fury.io/Nx-poGuiakLbsCzaLwNm/aexol -aexol==0.1.2 +aexol==0.1.3 From 9a2cbbd3cc06e570d644c8096dbae407731f7d81 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 25 May 2016 11:12:53 +0200 Subject: [PATCH 31/55] [INFRA-294]: temporary fix --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index e6e2abe..87e73b5 100644 --- a/test.py +++ b/test.py @@ -1,5 +1,5 @@ if __name__ == '__main__': import pkgutil modules = (name for _, name, is_pkg in pkgutil.iter_modules() - if is_pkg) + if is_pkg and name != 'aexol') map(__import__, modules) From 0dfbb3f7f7d6115a89227852dcb27f6e3ea3bc50 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 25 May 2016 13:38:00 +0200 Subject: [PATCH 32/55] [INFRA-294]: revert --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 87e73b5..e6e2abe 100644 --- a/test.py +++ b/test.py @@ -1,5 +1,5 @@ if __name__ == '__main__': import pkgutil modules = (name for _, name, is_pkg in pkgutil.iter_modules() - if is_pkg and name != 'aexol') + if is_pkg) map(__import__, modules) From 08a30bf935f2f763bb0cd307fd028df85f7189ff Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 25 May 2016 16:24:20 +0200 Subject: [PATCH 33/55] [INFRA-294]: version bump --- external_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external_requirements.txt b/external_requirements.txt index c5a4b74..ccb13ca 100644 --- a/external_requirements.txt +++ b/external_requirements.txt @@ -1,3 +1,3 @@ -i https://pypi.fury.io/Nx-poGuiakLbsCzaLwNm/aexol -aexol==0.1.3 +aexol==0.1.4 From 04bdb0fc0a824220b6291b94fe3174e787954c48 Mon Sep 17 00:00:00 2001 From: Jan Lipmann Date: Mon, 30 May 2016 10:49:59 +0200 Subject: [PATCH 34/55] sendinblue --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7f851df..3e3239d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ boto3==1.2.5 syncano==5.1.0 mixpanel==4.3.0 mixpanel-query-py==0.1.7 +Sendinblue==2.0.2 From 490722f8acf8c77f43fd4c086bd18602fc39c185 Mon Sep 17 00:00:00 2001 From: Maciej Kucharz Date: Sun, 5 Jun 2016 21:17:21 +0200 Subject: [PATCH 35/55] Adding GitPython. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3e3239d..3d3437b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ syncano==5.1.0 mixpanel==4.3.0 mixpanel-query-py==0.1.7 Sendinblue==2.0.2 +GitPython==2.0.5 From 9bf0cf47f5a6450955905e30f1fa7492354ba3d3 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Mon, 11 Jul 2016 07:17:53 +0200 Subject: [PATCH 36/55] [INFRA-329] bump ocr version --- external_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external_requirements.txt b/external_requirements.txt index ccb13ca..b3826da 100644 --- a/external_requirements.txt +++ b/external_requirements.txt @@ -1,3 +1,3 @@ -i https://pypi.fury.io/Nx-poGuiakLbsCzaLwNm/aexol -aexol==0.1.4 +aexol==0.1.5 From 49e107d702987972d96c5b5e9ab85658c27a6184 Mon Sep 17 00:00:00 2001 From: Maciej Kucharz Date: Mon, 11 Jul 2016 22:27:07 +0200 Subject: [PATCH 37/55] I need Mako! --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3d3437b..a332e4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ mixpanel==4.3.0 mixpanel-query-py==0.1.7 Sendinblue==2.0.2 GitPython==2.0.5 +Mako==1.0.4 From 6e7ef601f4bb69788055de336a18446254ded8bc Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Thu, 4 Aug 2016 12:28:08 -0400 Subject: [PATCH 38/55] Update googleads library Old one is being deprecated by Google. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a332e4e..a573bb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ oauthlib==1.0.3 requests-oauthlib==0.5.0 tweepy==3.4.0 braintree==3.20.0 -googleads==3.11.0 +googleads==4.3.0 fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 From 7d14ccabe506f105335cebeb5255a1b434667beb Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Thu, 25 Aug 2016 13:23:43 -0400 Subject: [PATCH 39/55] Add new packages, upgrade google-api-python-client --- requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a573bb0..327287d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ twilio==4.4.0 qrcode==5.1 analytics-python==1.0.3 osa==0.1.6.6 -google-api-python-client==1.4.1 +google-api-python-client==1.5.2 six==1.9.0 uritemplate==0.6 oauth2client==1.4.11 @@ -29,3 +29,7 @@ mixpanel-query-py==0.1.7 Sendinblue==2.0.2 GitPython==2.0.5 Mako==1.0.4 +sendgrid==3.2.10 +python-telegram-bot==5.0.0 +Jinja2==2.8 +wit==4.1.0 From 0c6a7e9c094f0da2586cb9936faffcfafa6eefd1 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Mon, 29 Aug 2016 11:37:36 +0200 Subject: [PATCH 40/55] [add-libraries-chatbots] add python-tk --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index b4680c1..8540e86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,8 @@ RUN apt-get update && apt-get install -qqy \ libjpeg-dev \ python-dev \ python3-dev \ + python-tk \ + python3-tk \ python-numpy \ python3-numpy \ python-scipy \ From 25605df4d14316700a7dc1cf298d2242c6b9de92 Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Mon, 29 Aug 2016 12:16:21 +0200 Subject: [PATCH 41/55] [add-libraries-chatbots] exclude winreg lib --- test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test.py b/test.py index e6e2abe..a994b27 100644 --- a/test.py +++ b/test.py @@ -1,5 +1,6 @@ if __name__ == '__main__': import pkgutil + blacklist = ['winreg'] modules = (name for _, name, is_pkg in pkgutil.iter_modules() - if is_pkg) + if is_pkg and name not in blacklist) map(__import__, modules) From f7eb9d697b1734611a8e663450db545f0d721537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Wed, 31 Aug 2016 13:05:56 +0200 Subject: [PATCH 42/55] [CORE-1649] update python Library version; --- .gitignore | 2 ++ requirements.txt | 2 +- requirements_python3.txt | 2 +- test_v50.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..92f3c73 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea/* +*.pyc diff --git a/requirements.txt b/requirements.txt index 327287d..492501f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==5.1.0 +syncano==5.4.1 mixpanel==4.3.0 mixpanel-query-py==0.1.7 Sendinblue==2.0.2 diff --git a/requirements_python3.txt b/requirements_python3.txt index b33554d..9092aef 100644 --- a/requirements_python3.txt +++ b/requirements_python3.txt @@ -23,6 +23,6 @@ fastly==0.0.2 pandas==0.17.1 beautifulsoup4==4.4.1 boto3==1.2.5 -syncano==5.1.0 +syncano==5.4.1 mixpanel==4.3.0 mixpanel-query-py==0.1.7 diff --git a/test_v50.py b/test_v50.py index 445efb5..65e939d 100644 --- a/test_v50.py +++ b/test_v50.py @@ -1,2 +1,2 @@ -for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.1.0')): +for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.4.1')): assert __import__(lib).__version__ == version From 0d1b70f084c87db52fe7f2db46fe174fcffc8ce7 Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Mon, 5 Sep 2016 17:57:32 -0400 Subject: [PATCH 43/55] New bot packages, google-api upgrade Same as #54 for Python 3 runtime --- requirements_python3.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements_python3.txt b/requirements_python3.txt index 9092aef..5747a4b 100644 --- a/requirements_python3.txt +++ b/requirements_python3.txt @@ -10,7 +10,7 @@ twilio==4.4.0 qrcode==5.1 analytics-python==1.0.3 osa==0.1.6.6 -google-api-python-client==1.4.1 +google-api-python-client==1.5.2 six==1.9.0 uritemplate==0.6 oauth2client==1.4.11 @@ -25,4 +25,8 @@ beautifulsoup4==4.4.1 boto3==1.2.5 syncano==5.4.1 mixpanel==4.3.0 +sendgrid==3.2.10 +python-telegram-bot==5.0.0 +Jinja2==2.8 +wit==4.1.0 mixpanel-query-py==0.1.7 From e1f21151b5a7fdeac63e921b192fc024aa115f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 27 Sep 2016 16:09:02 +0200 Subject: [PATCH 44/55] [CORE-1697] bump syncano python lib version --- requirements.txt | 2 +- requirements_python3.txt | 2 +- test_v50.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 492501f..b56064b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==5.4.1 +syncano==5.4.4 mixpanel==4.3.0 mixpanel-query-py==0.1.7 Sendinblue==2.0.2 diff --git a/requirements_python3.txt b/requirements_python3.txt index 5747a4b..7af2a33 100644 --- a/requirements_python3.txt +++ b/requirements_python3.txt @@ -23,7 +23,7 @@ fastly==0.0.2 pandas==0.17.1 beautifulsoup4==4.4.1 boto3==1.2.5 -syncano==5.4.1 +syncano==5.4.4 mixpanel==4.3.0 sendgrid==3.2.10 python-telegram-bot==5.0.0 diff --git a/test_v50.py b/test_v50.py index 65e939d..521fbac 100644 --- a/test_v50.py +++ b/test_v50.py @@ -1,2 +1,2 @@ -for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.4.1')): +for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.4.4')): assert __import__(lib).__version__ == version From b80724b17ffd3d75e8d609a8b108bd207601f6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 27 Sep 2016 16:21:08 +0200 Subject: [PATCH 45/55] [CORE-1697] update Sendinblue package also; --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b56064b..de76d71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ boto3==1.2.5 syncano==5.4.4 mixpanel==4.3.0 mixpanel-query-py==0.1.7 -Sendinblue==2.0.2 +Sendinblue==2.0.4 GitPython==2.0.5 Mako==1.0.4 sendgrid==3.2.10 From fb60f7f6c1ea4152f05e11b4fb45c8391d5df1cc Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 5 Oct 2016 11:42:06 +0200 Subject: [PATCH 46/55] [INFRA-317] rewrite to use Ansible --- Dockerfile | 69 ++----------------- README.md | 4 +- circle.yml | 8 ++- create_python_image.sh | 13 ++++ image/environment | 1 + image/id_rsa | 51 ++++++++++++++ image/id_rsa.pub | 1 + provision-python-codebox.yml | 5 ++ .../files/external_requirements.txt | 1 - python/files/pkglist | 15 ++++ .../files/requirements.txt | 0 .../files/requirements_base.txt | 0 .../files/requirements_python3.txt | 0 .../files/requirements_v42.txt | 0 python/tasks/common.yml | 38 ++++++++++ python/tasks/envs.yml | 43 ++++++++++++ python/tasks/main.yml | 24 +++++++ 17 files changed, 204 insertions(+), 69 deletions(-) create mode 100755 create_python_image.sh create mode 100644 image/environment create mode 100644 image/id_rsa create mode 100644 image/id_rsa.pub create mode 100644 provision-python-codebox.yml rename external_requirements.txt => python/files/external_requirements.txt (98%) create mode 100644 python/files/pkglist rename requirements.txt => python/files/requirements.txt (100%) rename requirements_base.txt => python/files/requirements_base.txt (100%) rename requirements_python3.txt => python/files/requirements_python3.txt (100%) rename requirements_v42.txt => python/files/requirements_v42.txt (100%) create mode 100644 python/tasks/common.yml create mode 100644 python/tasks/envs.yml create mode 100644 python/tasks/main.yml diff --git a/Dockerfile b/Dockerfile index 8540e86..846fd37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,66 +1,9 @@ FROM ubuntu:trusty MAINTAINER "Syncano DevOps Team" -ENV LAST_REFRESHED 2016-03-25 -ENV SYNCANO_APIROOT https://api.syncano.io/ - -RUN groupadd -r syncano && \ - useradd -u 1000 -r -g syncano syncano -d /tmp -s /bin/bash && \ - mkdir /home/syncano && \ - chown -R syncano /home/syncano - -# enable everyone to use /tmp -RUN chmod 1777 /tmp -# -- CUT BEGIN -- - -COPY *requirements*.txt /tmp/ -COPY *.tar.gz /tmp/ - -RUN apt-get update && apt-get install -qqy \ - git \ - libffi-dev \ - libssl-dev \ - libjpeg-dev \ - python-dev \ - python3-dev \ - python-tk \ - python3-tk \ - python-numpy \ - python3-numpy \ - python-scipy \ - python3-scipy \ - wget && \ - wget https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py && \ - pip install -r /tmp/requirements_base.txt - -WORKDIR /home/syncano/ -RUN virtualenv --system-site-packages v4.2 && \ - tar xzvf /tmp/42.tar.gz --skip-old-files && \ - . v4.2/bin/activate && \ - pip install -r /tmp/requirements_v42.txt && \ - pip install -r /tmp/external_requirements.txt && \ - deactivate - -RUN virtualenv --system-site-packages v5.0 && \ - tar xzvf /tmp/50.tar.gz --skip-old-files && \ - . v5.0/bin/activate && \ - pip install -r /tmp/requirements.txt && \ - pip install -r /tmp/external_requirements.txt && \ - deactivate - -RUN virtualenv --system-site-packages -p python3 p3v5.0 && \ - . p3v5.0/bin/activate && \ - pip install -r /tmp/requirements_python3.txt && \ - pip install -r /tmp/external_requirements.txt - -RUN ln -sf /home/syncano/v5.0/bin/python /usr/bin/python && \ - ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-lib5.0 && \ - ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 && \ - ln -sf /home/syncano/p3v5.0/bin/python /usr/bin/python3 && \ - ln -sf /home/syncano/p3v5.0/bin/python /usr/bin/python3-lib5.0 - -# -- CUT END -- -USER syncano -WORKDIR /tmp -CMD "python" +RUN apt-get update && apt-get install -y openssh-server +RUN mkdir /var/run/sshd /root/.ssh +COPY image/id_rsa.pub /root/.ssh/authorized_keys +RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.0.3/dumb-init_1.0.3_amd64.deb +RUN dpkg -i dumb-init_*.deb +EXPOSE 22 diff --git a/README.md b/README.md index 9d6f9c1..b9b6be7 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ This repository contains Dockerfile for Python 2.7 image with built-in support for Syncano and selected Python libraries. -You can build the image yourself: +You can build the image yourself (Ansible 2.1.0 is required): ``` -$ docker build -t quay.io/syncano/python-codebox . +$ ./create_python_image.sh ``` or pull it from Docker registry: diff --git a/circle.yml b/circle.yml index 2ea99f5..6f7c766 100644 --- a/circle.yml +++ b/circle.yml @@ -4,9 +4,11 @@ machine: dependencies: override: - - wget https://s3.amazonaws.com/codebox-bucket/python/42.tar.gz - - wget https://s3.amazonaws.com/codebox-bucket/python/50.tar.gz - - docker build -t quay.io/syncano/python-codebox . + - sudo pip install ansible==2.1.0 + - chmod 600 image/id_rsa + - ./create_python_image.sh $BUILD_LEVEL + - docker images + - docker history quay.io/syncano/python-codebox test: override: diff --git a/create_python_image.sh b/create_python_image.sh new file mode 100755 index 0000000..0a12684 --- /dev/null +++ b/create_python_image.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ $1 != "--partial" ] +then + docker build -t quay.io/syncano/python-codebox . +fi + +docker run -d -p 2200:22 --env-file image/environment --name ansible quay.io/syncano/python-codebox dumb-init /usr/sbin/sshd -D +ansible-playbook -i "`docker inspect --format '{{ .NetworkSettings.IPAddress }}' ansible`," -u root --private-key image/id_rsa provision-python-codebox.yml +docker commit ansible quay.io/syncano/python-codebox + +docker run -d --env-file image/environment -u syncano --name python quay.io/syncano/python-codebox dumb-init /usr/sbin/sshd -D +docker commit python quay.io/syncano/python-codebox diff --git a/image/environment b/image/environment new file mode 100644 index 0000000..7228805 --- /dev/null +++ b/image/environment @@ -0,0 +1 @@ +SYNCANO_APIROOT=https://api.syncano.io/ diff --git a/image/id_rsa b/image/id_rsa new file mode 100644 index 0000000..1f1b49a --- /dev/null +++ b/image/id_rsa @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAqdVP3R97ex0midPnCFkya/AWZG+tNDjHlHarTwEWjenxYfJu +eJTpUCxCFVomhVQVwdHQ9egdyy6KlsxC7fhPKNkLkkv+rCNk8Fn4m3NjmIv5eGN/ +YxpojxjNivfxK0PaTB9FrnZmJQNW/vLPuq92gl2OeNHfwIx6B4/yCxQtN1hTfA5f +6DKWorD874Z32pwJiJEnNygQXoHfXZdv4+pmbOLAKYjKMwNm+epy+CD0rwrChm/K +b9hcV9m9Z5s0qW5U4cqPBqt3QZghKwsj1g0rYQc0pHFg50NvWOQePJ3GCxstVfy5 +uAk7HIQj15xncb68xWmFaZLLYPxLeg4LMdEsTwejxpob1p1vtCGLILVFTilxQCOx +fJCRXaNDPQMwxHRyTSvJ5X6JvXlvs7bWsK3JUGnmzSa4s9B/N44/OrNjLW37AVx7 +j7Bo1GbBoqNfAYZ3GYvKfBTIcZ5fSUwH5A7mDSPDgBPLyUOJUg4ymZpyKilZASIV +Tp9weGU0BMoHnIFBag/Jd66Y98jCWjpHvCGKQUD9eIFUc7TRwtiI8VVD3+Fi6Yc9 +S8lCB1O5EiLD4os/Dmq1+F/YiHzTWnPEpVxuZw+vyzzQvzgroetHK3kWGwn9iDB2 +ABKin1GBjrQpWc0sH5au6NX0dpDL1RfaXHRk/m96kr6ZLTBeA9Z6gwsrTIMCAwEA +AQKCAgB/ZsipfXN5MK33WOIITjlX5Vw27dvFwA5f3gG2m39a6DMuodmjVToM7ZaJ +Q/5/qEtGAWq6/exTvk8yvno64Dopz2Ax2BZs9YREFYo8uvvstJxpZmJqaqYjKp0O +cd8V8KmyzwbL0H/hpja9vFJ+nh/NvfchxXcPj5H/WElfbDxbN0QxXOsvWbvMwe6M +lJgWg1VXgGQlqUVWUCDKqc7C79T0O8mhnr7oM9YtBmZGoxpKsUyp4vS+oFTthibi +sPh5XY4Jb5WJkFk++KtxCcNRUMHI4Mnpzi++bMT7ruzL1iN5s2J4DcUYPLx+MNZe +aTLkjgw4H2d4gilVUQ4huqVFMsAEM6WvB1a8mr0fXu0lHXmjIEVFS9wDX63FD8kR +TwrQMNn4hzZWGCj2i3vpkmzGPHkPK+rYQ61lkHyvN5IMTEsBIdvK6c09vMSIE8oc +Xm1X3aRShjgkVmRgrTKiLInHW1cKzVDANOW8jHx0g7IqQNe0ap899IyJsKcA4cXJ +fkRhyu9D0pxvkK6Jp65QR84yQ+qYDQQ+cVopg7PFtkj2Ek9s5Glq4FXEZXN1RU2Q +SlFm6eP3g+whMg+8hTBmnaBe0KmH7biJAGTTnL2xzCayXM/PtqrWOlCqtDoB1JVk +13r1P0+B4lPBitw2WXgEGk/neNt3Og2cLajsnT5oNGgdIjrFYQKCAQEA3horKk5U +fRbFRSBf5tWTRsoJrJ5F0ygp921ehhFHAH636eoDz8FuS/ZrAiLoIctmhBSY3H0b +cxpY2WiA3vao/GrBatQ+fi5mlwckIzCUoStHotVOisUl9AXhsUBjBYLsDYB8k1ni +89ZZC5x3XMdzoAy995U8b1Fcxyl5txuI8nlAOyPZf896tbKQ4zs/pLktaEetpo6W +QF7pgWGxfOsFBuej3q2AXm8h5Aw4xFv3LHP5KePM6m/xwZjsyzvNLWo3f0hRrLS/ +RYb4fzwfZ9WMe/Mh0Uv3a0r3k+Orpv159+8UDEqijpFo1hvmKU3q6fGVkxt5/J5y +GG1vZKYF3zDs0wKCAQEAw8Dssx327QWSKFRiCF9oWivWHGv+rQidjQd2MEkQF/9N +ab8cfBXNzEo9/jMcXtwa+bULDTuKtlTGsIqeXYrszRnFs91RYjSnrqbOFZJAHoxJ +/rBgvQjAmDddDyeRAaW1FU5U0uvqudDjS6ELBFIN79rXj5xJlgcC88C9nUmmHfaM +1HlIfN+LXJ5H5PkNgbD8ZT1ZJH4rEtlkPx5lNdgrShchrZ6ll13fLa+EbMOB0HNl +vFuQw1mMJpm3IEoAh1P/iXyWDpJRKTvKtld3AR0/4JOapqIUas+CIaaF/VZhwsUY +0dn+YASi2+bgWvI5VKfoaF9LGrws+oY8mxxdqqiTkQKCAQEAnalB3wQ/oZRacNms +ejY+F9Av0eOAeYZXVBbFNLJrDoTllRzTm9UZsL6584hH9EVo/nWIlWwR4kFftOUk +JiI+jwBRUL5dpkDV/nvJQVmpCEeq0IJPAN82M+VA8wxbvnvTiToeNJNoMeOEzmNB +sBC9mW2yJGBByFufWpmslzjqnAVKfTDTopDr8LNTepqCcTIqc43+TO+G5Sb65ycs +URqcNOyWPOo459BH5JRwb6NA8MNaqkvVGM5idSVTYmmPGp/9yVLS96BO7tIC8H8D +ZDgJQ6ux49rjRbyvxsGlBiOFQ2TpfFg20LGUE/k2BzgSfANMRGMK0zK+l2yH2JCq +NmeKlQKCAQEAugsLaxiZa58Vz7qTCT4T4U80SQLiovVDN5vV2gI2h3gezNfTbKMU +Wopc7NEKr6UlBhnojQT/ylY8F+FGrcSxViSeX1mCVYGRtE3YI2xCAEZ4tNRQvJgq +4wALM7H5AMXKmps9xtBs33kF7QT7gXmywTkOEfEJhsrXJW8TRv+OH4AJTI6QNA4J +V1cfISEKky3wUw/BPwuiSGHQAqUSr9/pFJtzaJ0U6A1RZQwhIXWWVNS35hJoPSFI +MA97mfZ7FiLhBHwh6WtqM3QeLrpl4es4oGnoW2JDYVgiZ3Tqd2G3u/Kyqg8y0XLW +jYSZyhXYIvsJyNh4espJPTiDNS/6XY2l4QKCAQEAzzRobY+LOLJUt1HAQdbJEeXS +4EH3vn7vPoxME56bSPvnz/w/u8RSh1mvnMx41ANYccUsNBbBdyZClG3UAToAxuxu +Z9j2ptNPFIlhZxGgf7o3p7hCU0DosMZVR1ADNJb+tGgXjHwctYIbX9Prbf30y06l +G3XlStZE0rVeC9RnM/OD4ymaXfp/wRsEGMX1vGhdsFebRFCauuimvRDOw8bRhUbI +oWXCUVSP5PqY/io0QAFZgCgi9OgtHO8qKzzK98bWPprHBXUGDWnuzXB/irbiQhBJ +tE6FDAh56fnBOxiYjdvh5iKv6rbuT3esoWe4S9H7vNjI7kHer95/tAsqVseTag== +-----END RSA PRIVATE KEY----- diff --git a/image/id_rsa.pub b/image/id_rsa.pub new file mode 100644 index 0000000..11354bd --- /dev/null +++ b/image/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCp1U/dH3t7HSaJ0+cIWTJr8BZkb600OMeUdqtPARaN6fFh8m54lOlQLEIVWiaFVBXB0dD16B3LLoqWzELt+E8o2QuSS/6sI2TwWfibc2OYi/l4Y39jGmiPGM2K9/ErQ9pMH0WudmYlA1b+8s+6r3aCXY540d/AjHoHj/ILFC03WFN8Dl/oMpaisPzvhnfanAmIkSc3KBBegd9dl2/j6mZs4sApiMozA2b56nL4IPSvCsKGb8pv2FxX2b1nmzSpblThyo8Gq3dBmCErCyPWDSthBzSkcWDnQ29Y5B48ncYLGy1V/Lm4CTschCPXnGdxvrzFaYVpkstg/Et6Dgsx0SxPB6PGmhvWnW+0IYsgtUVOKXFAI7F8kJFdo0M9AzDEdHJNK8nlfom9eW+zttawrclQaebNJriz0H83jj86s2MtbfsBXHuPsGjUZsGio18BhncZi8p8FMhxnl9JTAfkDuYNI8OAE8vJQ4lSDjKZmnIqKVkBIhVOn3B4ZTQEygecgUFqD8l3rpj3yMJaOke8IYpBQP14gVRztNHC2IjxVUPf4WLphz1LyUIHU7kSIsPiiz8OarX4X9iIfNNac8SlXG5nD6/LPNC/OCuh60creRYbCf2IMHYAEqKfUYGOtClZzSwflq7o1fR2kMvVF9pcdGT+b3qSvpktMF4D1nqDCytMgw== syncano diff --git a/provision-python-codebox.yml b/provision-python-codebox.yml new file mode 100644 index 0000000..62fdb24 --- /dev/null +++ b/provision-python-codebox.yml @@ -0,0 +1,5 @@ +--- +- hosts: all + connection: ssh + roles: + - python diff --git a/external_requirements.txt b/python/files/external_requirements.txt similarity index 98% rename from external_requirements.txt rename to python/files/external_requirements.txt index b3826da..8788358 100644 --- a/external_requirements.txt +++ b/python/files/external_requirements.txt @@ -1,3 +1,2 @@ -i https://pypi.fury.io/Nx-poGuiakLbsCzaLwNm/aexol aexol==0.1.5 - diff --git a/python/files/pkglist b/python/files/pkglist new file mode 100644 index 0000000..7c1c2ee --- /dev/null +++ b/python/files/pkglist @@ -0,0 +1,15 @@ +git +libffi-dev +libssl-dev +libjpeg-dev +python-dev +python3-dev +python-tk +python3-tk +python-numpy +python3-numpy +python-pip +python3-pip +python-scipy +python3-scipy +wget diff --git a/requirements.txt b/python/files/requirements.txt similarity index 100% rename from requirements.txt rename to python/files/requirements.txt diff --git a/requirements_base.txt b/python/files/requirements_base.txt similarity index 100% rename from requirements_base.txt rename to python/files/requirements_base.txt diff --git a/requirements_python3.txt b/python/files/requirements_python3.txt similarity index 100% rename from requirements_python3.txt rename to python/files/requirements_python3.txt diff --git a/requirements_v42.txt b/python/files/requirements_v42.txt similarity index 100% rename from requirements_v42.txt rename to python/files/requirements_v42.txt diff --git a/python/tasks/common.yml b/python/tasks/common.yml new file mode 100644 index 0000000..1f4fcb0 --- /dev/null +++ b/python/tasks/common.yml @@ -0,0 +1,38 @@ +--- +- name: Add syncano group + group: + name: syncano + gid: 1000 + state: present + +- name: Add syncano user + user: + name: syncano + group: syncano + uid: 1000 + shell: /bin/bash + home: /home/syncano + state: present + +- name: Copy apt pkgs + copy: + src: pkglist + dest: /tmp/ + +- name: List apt packages + shell: + cat /tmp/pkglist + register: pkgs + +- name: Install packages + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + "{{ pkgs.stdout_lines }}" + +- name: Allow public usage of /tmp + file: + name: /tmp + mode: "1777" \ No newline at end of file diff --git a/python/tasks/envs.yml b/python/tasks/envs.yml new file mode 100644 index 0000000..e35faad --- /dev/null +++ b/python/tasks/envs.yml @@ -0,0 +1,43 @@ +- name: Install Python packages in envs + pip: + virtualenv: "/home/syncano/{{ item.env }}" + virtualenv_python: "{{ item.python }}" + virtualenv_site_packages: yes + requirements: "/tmp/{{ item.requirements }}" + with_items: + - env: v4.2 + python: python2.7 + requirements: requirements_v42.txt + - env: v4.2 + python: python2.7 + requirements: external_requirements.txt + - env: v5.0 + python: python2.7 + requirements: requirements.txt + - env: v5.0 + python: python2.7 + requirements: external_requirements.txt + - env: p3v5.0 + python: python3 + requirements: requirements_python3.txt + - env: p3v5.0 + python: python3 + requirements: external_requirements.txt + +- name: Set soft links to runtimes + file: + src: "/home/syncano/{{ item.env }}/bin/python" + dest: "/usr/bin/{{ item.python }}" + state: link + force: yes + with_items: + - env: v5.0 + python: python + - env: v5.0 + python: python27-lib5.0 + - env: v4.2 + python: python27-lib4.2 + - env: p3v5.0 + python: python3 + - env: p3v5.0 + python: python3-lib5.0 diff --git a/python/tasks/main.yml b/python/tasks/main.yml new file mode 100644 index 0000000..d7cb095 --- /dev/null +++ b/python/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- include: common.yml + +- name: Install requirements + pip: + name: pip + extra_args: "--upgrade" + +- name: Copy requirements files + copy: + src: "{{ item }}" + dest: /tmp/ + with_items: + - requirements.txt + - requirements_base.txt + - requirements_v42.txt + - requirements_python3.txt + - external_requirements.txt + +- name: Install requirements + pip: + requirements: /tmp/requirements_base.txt + +- include: envs.yml From 7b5167449f73d1c7a3ffd59db9f36ed168156def Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 7 Oct 2016 11:44:25 +0200 Subject: [PATCH 47/55] [INFRA-317] correct default param [ci skip] --- create_python_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_python_image.sh b/create_python_image.sh index 0a12684..e66953b 100755 --- a/create_python_image.sh +++ b/create_python_image.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ $1 != "--partial" ] +if [ ${1:-full} != "--partial" ] then docker build -t quay.io/syncano/python-codebox . fi From c2c869a87ae3add943369c1141ba429d87fb0c1c Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Thu, 17 Nov 2016 15:37:39 +0100 Subject: [PATCH 48/55] [version_bump] Syncano version bump to 5.4.6 --- python/files/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/files/requirements.txt b/python/files/requirements.txt index de76d71..5cbac2d 100644 --- a/python/files/requirements.txt +++ b/python/files/requirements.txt @@ -23,7 +23,7 @@ fastly==0.0.2 pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==5.4.4 +syncano==5.4.6 mixpanel==4.3.0 mixpanel-query-py==0.1.7 Sendinblue==2.0.4 From 6f6e4ed4a191937787c02e806c71eece5263c129 Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Thu, 17 Nov 2016 15:38:23 +0100 Subject: [PATCH 49/55] [syncano_546] python3 update --- python/files/requirements_python3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/files/requirements_python3.txt b/python/files/requirements_python3.txt index 7af2a33..475eb13 100644 --- a/python/files/requirements_python3.txt +++ b/python/files/requirements_python3.txt @@ -23,7 +23,7 @@ fastly==0.0.2 pandas==0.17.1 beautifulsoup4==4.4.1 boto3==1.2.5 -syncano==5.4.4 +syncano==5.4.6 mixpanel==4.3.0 sendgrid==3.2.10 python-telegram-bot==5.0.0 From 93971e1dfee31dfcd7ee294813cb49b1026ac632 Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Thu, 17 Nov 2016 16:15:14 +0100 Subject: [PATCH 50/55] [syncano_546] Version bump --- test_v50.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_v50.py b/test_v50.py index 521fbac..e929731 100644 --- a/test_v50.py +++ b/test_v50.py @@ -1,2 +1,2 @@ -for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.4.4')): +for lib, version in (('numpy', '1.8.2'), ('scipy', '0.13.3'), ('syncano', '5.4.6')): assert __import__(lib).__version__ == version From 6428826a89703eaf5c6caa922124677f95a17f8a Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Thu, 17 Nov 2016 17:53:17 +0100 Subject: [PATCH 51/55] [syncano_546] explicit future requirement --- python/files/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/python/files/requirements.txt b/python/files/requirements.txt index 5cbac2d..9d6bbd3 100644 --- a/python/files/requirements.txt +++ b/python/files/requirements.txt @@ -33,3 +33,4 @@ sendgrid==3.2.10 python-telegram-bot==5.0.0 Jinja2==2.8 wit==4.1.0 +future==0.15.2 From d1134976dce7c14658023c5bf91621a3775d26ab Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Thu, 24 Nov 2016 14:48:36 +0100 Subject: [PATCH 52/55] [INFRA-425] add workdir and cmd --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 846fd37..bb55724 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,3 +7,6 @@ COPY image/id_rsa.pub /root/.ssh/authorized_keys RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.0.3/dumb-init_1.0.3_amd64.deb RUN dpkg -i dumb-init_*.deb EXPOSE 22 + +WORKDIR /tmp +CMD "python" From 37c696d41bf236358abfcb034853265d841c5c94 Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Mon, 9 Jan 2017 19:21:24 +0100 Subject: [PATCH 53/55] [sendinblue] Force updated Due to incredibly stupid policy on behalf of sendinblue (they remove older versions from PyPI when they release new ones). --- python/files/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/files/requirements.txt b/python/files/requirements.txt index 9d6bbd3..4a6be6c 100644 --- a/python/files/requirements.txt +++ b/python/files/requirements.txt @@ -26,7 +26,7 @@ boto3==1.2.5 syncano==5.4.6 mixpanel==4.3.0 mixpanel-query-py==0.1.7 -Sendinblue==2.0.4 +Sendinblue==2.0.5.1 GitPython==2.0.5 Mako==1.0.4 sendgrid==3.2.10 From 7b56db3647aec8d3714b3167aaa5161ca3f40686 Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Fri, 1 Jun 2018 21:35:46 +0200 Subject: [PATCH 54/55] fix: upgrade setuptools --- python/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/tasks/main.yml b/python/tasks/main.yml index d7cb095..6b0e8af 100644 --- a/python/tasks/main.yml +++ b/python/tasks/main.yml @@ -3,8 +3,11 @@ - name: Install requirements pip: - name: pip + name: " {{ item }}" extra_args: "--upgrade" + with_items: + - pip + - setuptools - name: Copy requirements files copy: From 927a38e6bacf662c2940d56b8f08b85fa92793a5 Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Fri, 1 Jun 2018 22:38:35 +0200 Subject: [PATCH 55/55] fix: fix reqs --- python/files/requirements.txt | 61 ++++++++++++++------------- python/files/requirements_python3.txt | 53 +++++++++++------------ python/files/requirements_v42.txt | 41 +++++++++--------- python/tasks/main.yml | 1 - 4 files changed, 79 insertions(+), 77 deletions(-) diff --git a/python/files/requirements.txt b/python/files/requirements.txt index 4a6be6c..0770e7f 100644 --- a/python/files/requirements.txt +++ b/python/files/requirements.txt @@ -1,36 +1,37 @@ -Pillow==2.7.0 -requests==2.7.0 -icalendar==3.8.4 -pytz==2014.10 -html2text==2015.2.18 -pkgtools==0.7.3 -names==0.3.0 -httplib2==0.9.1 -twilio==4.4.0 -qrcode==5.1 analytics-python==1.0.3 -osa==0.1.6.6 -google-api-python-client==1.5.2 -six==1.9.0 -uritemplate==0.6 -oauth2client==1.4.11 -oauthlib==1.0.3 -requests-oauthlib==0.5.0 -tweepy==3.4.0 -braintree==3.20.0 -googleads==4.3.0 -fastly==0.0.2 -pandas==0.17.1 BeautifulSoup==3.2.1 boto3==1.2.5 -syncano==5.4.6 -mixpanel==4.3.0 -mixpanel-query-py==0.1.7 -Sendinblue==2.0.5.1 +braintree==3.20.0 +cryptography==2.0.3 +fastly==0.0.2 +future==0.15.2 GitPython==2.0.5 +google-api-python-client==1.5.2 +googleads==4.3.0 +html2text==2015.2.18 +httplib2==0.9.1 +icalendar==3.8.4 +Jinja2==2.8 Mako==1.0.4 -sendgrid==3.2.10 +mixpanel-query-py==0.1.7 +mixpanel==4.3.0 +names==0.3.0 +oauth2client==1.4.11 +oauthlib==1.0.3 +osa==0.1.6.6 +pandas==0.17.1 +Pillow==2.7.0 +pkgtools==0.7.3 python-telegram-bot==5.0.0 -Jinja2==2.8 -wit==4.1.0 -future==0.15.2 +pytz==2014.10 +qrcode==5.1 +requests-oauthlib==0.5.0 +requests==2.7.0 +sendgrid==3.2.10 +Sendinblue==2.0.5.1 +six==1.9.0 +syncano==5.4.6 +tweepy==3.4.0 +twilio==4.4.0 +uritemplate==0.6 +wit==4.1.0 \ No newline at end of file diff --git a/python/files/requirements_python3.txt b/python/files/requirements_python3.txt index 475eb13..a55b8a9 100644 --- a/python/files/requirements_python3.txt +++ b/python/files/requirements_python3.txt @@ -1,32 +1,33 @@ -Pillow==2.7.0 -requests==2.7.0 -icalendar==3.8.4 -pytz==2014.10 -html2text==2015.2.18 -pkgtools==0.7.3 -names==0.3.0 -httplib2==0.9.1 -twilio==4.4.0 -qrcode==5.1 analytics-python==1.0.3 -osa==0.1.6.6 +beautifulsoup4==4.4.1 +boto3==1.2.5 +braintree==3.20.0 +cryptography==2.0.3 +fastly==0.0.2 google-api-python-client==1.5.2 -six==1.9.0 -uritemplate==0.6 +googleads==3.11.0 +html2text==2015.2.18 +httplib2==0.9.1 +icalendar==3.8.4 +Jinja2==2.8 +mixpanel-query-py==0.1.7 +mixpanel==4.3.0 +names==0.3.0 oauth2client==1.4.11 oauthlib==1.0.3 -requests-oauthlib==0.5.0 -tweepy==3.4.0 -braintree==3.20.0 -googleads==3.11.0 -fastly==0.0.2 +osa==0.1.6.6 pandas==0.17.1 -beautifulsoup4==4.4.1 -boto3==1.2.5 -syncano==5.4.6 -mixpanel==4.3.0 -sendgrid==3.2.10 +Pillow==2.7.0 +pkgtools==0.7.3 python-telegram-bot==5.0.0 -Jinja2==2.8 -wit==4.1.0 -mixpanel-query-py==0.1.7 +pytz==2014.10 +qrcode==5.1 +requests-oauthlib==0.5.0 +requests==2.7.0 +sendgrid==3.2.10 +six==1.9.0 +syncano==5.4.6 +tweepy==3.4.0 +twilio==4.4.0 +uritemplate==0.6 +wit==4.1.0 \ No newline at end of file diff --git a/python/files/requirements_v42.txt b/python/files/requirements_v42.txt index b711e79..f04ff33 100644 --- a/python/files/requirements_v42.txt +++ b/python/files/requirements_v42.txt @@ -1,26 +1,27 @@ -Pillow==2.7.0 -requests==2.7.0 -icalendar==3.8.4 -pytz==2014.10 -html2text==2015.2.18 -pkgtools==0.7.3 -names==0.3.0 -httplib2==0.9.1 -twilio==4.4.0 -qrcode==5.1 analytics-python==1.0.3 -osa==0.1.6.6 +BeautifulSoup==3.2.1 +boto3==1.2.5 +braintree==3.20.0 +cryptography==2.0.3 +fastly==0.0.2 google-api-python-client==1.4.1 -six==1.9.0 -uritemplate==0.6 +googleads==3.11.0 +html2text==2015.2.18 +httplib2==0.9.1 +icalendar==3.8.4 +names==0.3.0 oauth2client==1.4.11 oauthlib==1.0.3 -requests-oauthlib==0.5.0 -tweepy==3.4.0 -braintree==3.20.0 -googleads==3.11.0 -fastly==0.0.2 +osa==0.1.6.6 pandas==0.17.1 -BeautifulSoup==3.2.1 -boto3==1.2.5 +Pillow==2.7.0 +pkgtools==0.7.3 +pytz==2014.10 +qrcode==5.1 +requests-oauthlib==0.5.0 +requests==2.7.0 +six==1.9.0 syncano==4.2.0 +tweepy==3.4.0 +twilio==4.4.0 +uritemplate==0.6 \ No newline at end of file diff --git a/python/tasks/main.yml b/python/tasks/main.yml index 6b0e8af..0a75bf1 100644 --- a/python/tasks/main.yml +++ b/python/tasks/main.yml @@ -7,7 +7,6 @@ extra_args: "--upgrade" with_items: - pip - - setuptools - name: Copy requirements files copy: