Skip to content

Commit 8ee2d68

Browse files
Danny Joycedwillist
Danny Joyce
and
dwillist
committed
Handle recursive requirements in requirements.txt
- Do not copy requirements.txt to the deps dir - Generate and clean up a temp requirements.txt when required - Run pip install using the requirements.txt in the app dir - Minor formatting and error log changes [#158115122] Co-authored-by: Daniel Thornton <dthornton@pivotal.io>
1 parent 6957c7b commit 8ee2d68

9 files changed

Lines changed: 115 additions & 62 deletions

File tree

‎fixtures/recursive_requirements/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.pyc
2+
venv

‎fixtures/recursive_requirements/Procfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
web: gunicorn server:app

‎fixtures/recursive_requirements/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
flask==0.10.1
2+
gunicorn==19.3.0
3+
itsdangerous==0.24
4+
jinja2==2.7.2
5+
-r sub_folder/other_requirement1.txt

‎fixtures/recursive_requirements/server.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from flask import Flask, request
2+
import subprocess
3+
4+
app = Flask(__name__)
5+
6+
@app.route("/")
7+
def hello():
8+
return "Hello, World!"
9+
10+
@app.route('/execute', methods=['POST'])
11+
def execute():
12+
with open('runtime.py', 'w') as f:
13+
f.write(request.values.get('code'))
14+
return subprocess.check_output(["python", "runtime.py"])
15+
16+
app.debug=True

‎fixtures/recursive_requirements/sub_folder/other_requirement1.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
markupsafe==0.21
2+
-r other_requirements2.txt

‎fixtures/recursive_requirements/sub_folder/other_requirements2.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
werkzeug==0.10.4

‎src/python/integration/deploy_a_python_app_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,12 @@ var _ = Describe("CF Python Buildpack", func() {
5757
})
5858
})
5959

60+
It("deploy a web app with -r in requirements.txt", func() {
61+
app = cutlass.New(filepath.Join(bpDir, "fixtures", "recursive_requirements"))
62+
PushAppAndConfirm(app)
63+
Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
64+
})
65+
6066
It("deploy a web app that uses an nltk corpus", func() {
6167
app = cutlass.New(filepath.Join(bpDir, "fixtures", "nltk_flask"))
6268
app.SetEnv("BP_DEBUG", "1")

‎src/python/supply/supply.go‎

Lines changed: 54 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ type Supplier struct {
5555
Log *libbuildpack.Logger
5656
Logfile *os.File
5757
HasNltkData bool
58+
removeRequirementsText bool
5859
}
5960

6061
func Run(s *Supplier) error {
@@ -78,8 +79,8 @@ func RunPython(s *Supplier) error {
7879
return err
7980
}
8081

81-
if err := s.CopyRequirementsAndRuntimeTxt(); err != nil {
82-
s.Log.Error("Error copying requirements.txt and runtime.txt to deps dir: %v", err)
82+
if err := s.CopyRuntimeTxt(); err != nil {
83+
s.Log.Error("Error copying runtime.txt to deps dir: %v", err)
8384
return err
8485
}
8586

@@ -157,19 +158,19 @@ func RunPython(s *Supplier) error {
157158
s.Log.Debug("Size of pip cache dir: %s", cacheDirSize)
158159
}
159160

161+
if s.removeRequirementsText {
162+
if err := os.Remove(filepath.Join(s.Stager.BuildDir(), "requirements.txt")); err != nil {
163+
s.Log.Error("Unable to clean up app directory: %s", err.Error())
164+
return err
165+
}
166+
}
167+
160168
dirSnapshot.Diff()
161169

162170
return nil
163171
}
164172

165-
func (s *Supplier) CopyRequirementsAndRuntimeTxt() error {
166-
if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "requirements.txt")); err != nil {
167-
return err
168-
} else if exists {
169-
if err = libbuildpack.CopyFile(filepath.Join(s.Stager.BuildDir(), "requirements.txt"), filepath.Join(s.Stager.DepDir(), "requirements.txt")); err != nil {
170-
return err
171-
}
172-
}
173+
func (s *Supplier) CopyRuntimeTxt() error {
173174
if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "runtime.txt")); err != nil {
174175
return err
175176
} else if exists {
@@ -181,7 +182,7 @@ func (s *Supplier) CopyRequirementsAndRuntimeTxt() error {
181182
}
182183

183184
func (s *Supplier) HandleMercurial() error {
184-
if err := s.Command.Execute(s.Stager.DepDir(), ioutil.Discard, ioutil.Discard, "grep", "-Fiq", "hg+", "requirements.txt"); err != nil {
185+
if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "grep", "-Fiq", "hg+", "requirements.txt"); err != nil {
185186
return nil
186187
}
187188

@@ -336,7 +337,7 @@ func (s *Supplier) InstallPipPop() error {
336337
}
337338

338339
func (s *Supplier) InstallPipEnv() error {
339-
requirementstxtExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.DepDir(), "requirements.txt"))
340+
requirementstxtExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "requirements.txt"))
340341
if err != nil {
341342
return err
342343
}
@@ -385,16 +386,15 @@ func (s *Supplier) InstallPipEnv() error {
385386
}
386387
}
387388

388-
if err := ioutil.WriteFile(filepath.Join(s.Stager.DepDir(), "requirements.txt"), []byte(outputString), 0644); err != nil {
389-
return err
390-
}
389+
return s.writeTempRequirementsTxt()
391390
}
392391
return nil
393392
}
394393

395394
func (s *Supplier) HandlePylibmc() error {
396395
memcachedDir := filepath.Join(s.Stager.DepDir(), "libmemcache")
397-
if err := s.Command.Execute(s.Stager.DepDir(), ioutil.Discard, ioutil.Discard, "pip-grep", "-s", "requirements.txt", "pylibmc"); err == nil {
396+
397+
if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "pip-grep", "-s", "requirements.txt", "pylibmc"); err == nil {
398398
s.Log.BeginStep("Noticed pylibmc. Bootstrapping libmemcached.")
399399
if err := s.Installer.InstallOnlyVersion("libmemcache", memcachedDir); err != nil {
400400
return err
@@ -411,7 +411,7 @@ func (s *Supplier) HandlePylibmc() error {
411411
}
412412

413413
func (s *Supplier) HandleRequirementstxt() error {
414-
if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.DepDir(), "requirements.txt")); err != nil {
414+
if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "requirements.txt")); err != nil {
415415
return err
416416
} else if exists {
417417
return nil
@@ -423,7 +423,7 @@ func (s *Supplier) HandleRequirementstxt() error {
423423
return nil
424424
}
425425

426-
return ioutil.WriteFile(filepath.Join(s.Stager.DepDir(), "requirements.txt"), []byte("-e ."), 0644)
426+
return s.writeTempRequirementsTxt()
427427
}
428428

429429
func (s *Supplier) installFfi() error {
@@ -449,7 +449,7 @@ func (s *Supplier) installFfi() error {
449449
}
450450

451451
func (s *Supplier) HandleFfi() error {
452-
if err := s.Command.Execute(s.Stager.DepDir(), ioutil.Discard, ioutil.Discard, "pip-grep", "-s", "requirements.txt", "argon2-cffi", "bcrypt", "cffi", "cryptography", "django[argon2]", "Django[argon2]", "django[bcrypt]", "Django[bcrypt]", "PyNaCl", "pyOpenSSL", "PyOpenSSL", "requests[security]", "misaka"); err == nil {
452+
if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "pip-grep", "-s", "requirements.txt", "argon2-cffi", "bcrypt", "cffi", "cryptography", "django[argon2]", "Django[argon2]", "django[bcrypt]", "Django[bcrypt]", "PyNaCl", "pyOpenSSL", "PyOpenSSL", "requests[security]", "misaka"); err == nil {
453453
return s.installFfi()
454454
}
455455
return nil
@@ -490,7 +490,17 @@ func (s *Supplier) UninstallUnusedDependencies() error {
490490
fileContents, _ := ioutil.ReadFile(filepath.Join(s.Stager.DepDir(), "python", "requirements-declared.txt"))
491491
s.Log.Info("requirements-declared: %s", string(fileContents))
492492

493-
staleContents, err := s.Command.Output(s.Stager.BuildDir(), "pip-diff", "--stale", filepath.Join(s.Stager.DepDir(), "python", "requirements-declared.txt"), filepath.Join(s.Stager.DepDir(), "requirements.txt"), "--exclude", "setuptools", "pip", "wheel")
493+
staleContents, err := s.Command.Output(
494+
s.Stager.BuildDir(),
495+
"pip-diff",
496+
"--stale",
497+
filepath.Join(s.Stager.DepDir(), "python", "requirements-declared.txt"),
498+
filepath.Join(s.Stager.BuildDir(), "requirements.txt"),
499+
"--exclude",
500+
"setuptools",
501+
"pip",
502+
"wheel",
503+
)
494504
if err != nil {
495505
return err
496506
}
@@ -504,7 +514,15 @@ func (s *Supplier) UninstallUnusedDependencies() error {
504514
}
505515

506516
s.Log.BeginStep("Uninstalling stale dependencies")
507-
if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "pip", "uninstall", "-r", filepath.Join(s.Stager.DepDir(), "python", "requirements-stale.txt", "-y", "--exists-action=w")); err != nil {
517+
if err := s.Command.Execute(
518+
s.Stager.BuildDir(),
519+
indentWriter(os.Stdout),
520+
indentWriter(os.Stderr),
521+
"pip",
522+
"uninstall",
523+
"-r",
524+
filepath.Join(s.Stager.DepDir(), "python", "requirements-stale.txt", "-y", "--exists-action=w"),
525+
); err != nil {
508526
return err
509527
}
510528

@@ -516,17 +534,17 @@ func (s *Supplier) UninstallUnusedDependencies() error {
516534
func (s *Supplier) RunPip() error {
517535
s.Log.BeginStep("Running Pip Install")
518536

519-
requirementsPath := filepath.Join(s.Stager.DepDir(), "requirements.txt")
537+
requirementsPath := filepath.Join(s.Stager.BuildDir(), "requirements.txt")
520538
if exists, err := libbuildpack.FileExists(requirementsPath); err != nil {
521-
return fmt.Errorf("Couldn't determine existence of requirements.txt: %v", err)
539+
return fmt.Errorf("could not determine existence of requirements.txt: %v", err)
522540
} else if !exists {
523541
s.Log.Debug("Skipping 'pip install' since requirements.txt does not exist")
524542
return nil
525543
}
526544

527545
vendorExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "vendor"))
528546
if err != nil {
529-
return fmt.Errorf("Couldn't check vendor existence: %v", err)
547+
return fmt.Errorf("could not check vendor existence: %v", err)
530548
}
531549

532550
installArgs := []string{"install", "-r", requirementsPath, "--ignore-installed", "--exists-action=w", "--src=" + filepath.Join(s.Stager.DepDir(), "src")}
@@ -539,30 +557,30 @@ func (s *Supplier) RunPip() error {
539557
// and pipenv generates requirements.txt with -i
540558
originalReqs, err = ioutil.ReadFile(requirementsPath)
541559
if err != nil {
542-
return fmt.Errorf("Could not read requirements.txt: %v", err)
560+
return fmt.Errorf("could not read requirements.txt: %v", err)
543561
}
544562
re := regexp.MustCompile("(?m)^\\W*-i.*$")
545563
modifiedReqs := re.ReplaceAll(originalReqs, []byte{})
546564
err = ioutil.WriteFile(requirementsPath, modifiedReqs, 0644)
547565
if err != nil {
548-
return fmt.Errorf("Could not overwrite requirements file: %v", err)
566+
return fmt.Errorf("could not overwrite requirements file: %v", err)
549567
}
550568
}
551569

552570
if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "pip", installArgs...); err != nil {
553571
if vendorExists {
554572
s.Log.Info("Running pip install without indexes failed. Not all dependencies were vendored. Trying again with indexes.")
555573

556-
err = ioutil.WriteFile(requirementsPath, originalReqs, 0644)
557-
if err != nil {
558-
return fmt.Errorf("Could not overwrite modified requirements file: %v", err)
574+
if err := ioutil.WriteFile(requirementsPath, originalReqs, 0644); err != nil {
575+
return fmt.Errorf("could not overwrite modified requirements file: %v", err)
559576
}
577+
560578
if err = s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "pip", installArgs...); err != nil {
561579
s.Log.Info("Running pip install failed. You need to include all dependencies in the vendor directory.")
562-
return fmt.Errorf("Couldn't run pip: %v", err)
580+
return fmt.Errorf("could not run pip: %v", err)
563581
}
564582
} else {
565-
return fmt.Errorf("Couldn't run pip: %v", err)
583+
return fmt.Errorf("could not run pip: %v", err)
566584
}
567585
}
568586

@@ -646,6 +664,11 @@ func (s *Supplier) formatVersion(version string) string {
646664

647665
}
648666

667+
func (s *Supplier) writeTempRequirementsTxt() error {
668+
s.removeRequirementsText = true
669+
return ioutil.WriteFile(filepath.Join(s.Stager.BuildDir(), "requirements.txt"), []byte("-e ."), 0644)
670+
}
671+
649672
func indentWriter(writer io.Writer) io.Writer {
650673
return text.NewIndentWriter(writer, []byte(" "))
651674
}

0 commit comments

Comments
 (0)