Allow requirements_in to be generated (bazel-contrib#829)

philsc · f0rmiga · web-flow · commit b15e15fb95d5 · 2022-10-05T16:41:36.000-07:00
* Allow requirements_in to be generated

We generate the `requirements_in` file from various input files
roughly like so:

    genrule(
        name = "generate_3.7_x86_requirements",
        srcs = [
            "requirements_base.in.txt",
            "requirements_extra_37.in.txt",
        ],
        outs = ["requirements_3.7_x86.txt"],
        cmd = "cat $(SRCS) &gt; $(OUTS)",
    )

    compile_pip_requirements(
        name = "compile_requirements_3.7_x86",
        requirements_in = ":requirements_3.7_x86.txt",
        requirements_txt = "requirements_3.7_x86.lock.txt",
    )

The current code errors out with a message like this:

    Updating common/python/requirements_3.7_x86.lock.txt
    Usage: pip_compile.py [OPTIONS] [SRC_FILES]...
    Try 'pip_compile.py -h' for help.

    Error: Invalid value for '[SRC_FILES]...': Path 'common/python/requirements_3.7_x86.txt' does not exist.

This patch here fixes the issue by resolving the `requirements_in`
path before the tool `cd`s into the workspace directory.

* Make tests pass

* Run black

* Fix some runtime problems

Co-authored-by: Thulio Ferraz Assis &lt;3149049+f0rmiga@users.noreply.github.com&gt;
diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD
@@ -55,6 +55,16 @@ alias(
     actual = entry_point("yamllint"),
 )
 
+# The requirements.in file can be checked in to the source tree or it can be
+# generated. Pretend that we do some generating of the file. For this example,
+# the "template" is already the file we want.
+genrule(
+    name = "generate_requirements_in",
+    srcs = ["requirements.in.template"],
+    outs = ["requirements.in"],
+    cmd = "cp $(SRCS) $(OUTS)",
+)
+
 # This rule adds a convenient way to update the requirements file.
 compile_pip_requirements(
     name = "requirements",
diff --git a/examples/pip_parse/requirements.in.template b/examples/pip_parse/requirements.in.template
diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py
@@ -2,6 +2,7 @@
 
 import os
 import sys
+from pathlib import Path
 from shutil import copyfile
 
 from piptools.scripts.compile import cli
@@ -25,6 +26,21 @@ def _select_golden_requirements_file(
         return requirements_txt
 
 
+def _fix_up_requirements_in_path(
+    resolved_requirements_in, requirements_in, output_file
+):
+    """Fix up references to the input file inside of the generated requirements file.
+
+    We don't want fully resolved, absolute paths in the generated requirements file.
+    The paths could differ for every invocation. Replace them with a predictable path.
+    """
+    output_file = Path(output_file)
+    fixed_requirements_text = output_file.read_text().replace(
+        resolved_requirements_in, requirements_in
+    )
+    output_file.write_text(fixed_requirements_text)
+
+
 if __name__ == "__main__":
     if len(sys.argv) < 4:
         print(
@@ -42,6 +58,10 @@ def _select_golden_requirements_file(
     requirements_windows = parse_str_none(sys.argv.pop(1))
     update_target_label = sys.argv.pop(1)
 
+    # The requirements_in file could be generated. We need to get the path to it before we change
+    # directory into the workspace directory.
+    resolved_requirements_in = str(Path(requirements_in).resolve())
+
     # Before loading click, set the locale for its parser.
     # If it leaks through to the system setting, it may fail:
     # RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII
@@ -96,11 +116,18 @@ def _select_golden_requirements_file(
     sys.argv.append("--generate-hashes")
     sys.argv.append("--output-file")
     sys.argv.append(requirements_txt if UPDATE else requirements_out)
-    sys.argv.append(requirements_in)
+    sys.argv.append(resolved_requirements_in)
 
     if UPDATE:
         print("Updating " + requirements_txt)
-        cli()
+        try:
+            cli()
+        except SystemExit as e:
+            if e.code == 0:
+                _fix_up_requirements_in_path(
+                    resolved_requirements_in, requirements_in, requirements_txt
+                )
+            raise
     else:
         # cli will exit(0) on success
         try:
@@ -118,6 +145,9 @@ def _select_golden_requirements_file(
                 )
                 sys.exit(1)
             elif e.code == 0:
+                _fix_up_requirements_in_path(
+                    resolved_requirements_in, requirements_in, requirements_out
+                )
                 golden_filename = _select_golden_requirements_file(
                     requirements_txt,
                     requirements_linux,
