Doug/add debug find files (SocketDev#17)

dacoburn · web-flow · commit 6d4e171c2ad1 · 2024-08-12T10:43:28.000-07:00
* Added debug logic to find files

* Added more debug logging for find files

* Changes to the find_files function to reduce memory usage
diff --git a/.gitignore b/.gitignore
@@ -18,4 +18,5 @@ markdown_security_temp.md
 .DS_Store
 *.pyc
 test.py
-*.cpython-312.pyc
+*.cpython-312.pyc`
+file_generator.py
diff --git a/pyproject.toml b/pyproject.toml
@@ -11,7 +11,8 @@ dependencies = [
     'mdutils',
     'prettytable',
     'argparse',
-    'GitPython'
+    'GitPython',
+    'packaging'
 ]
 readme = "README.md"
 description = "Socket Security CLI for CI/CD"
diff --git a/requirements.txt b/requirements.txt
@@ -2,4 +2,5 @@ requests>=2.32.0
 mdutils~=1.6.0
 prettytable
 argparse
-gitpython>=3.1.43
+gitpython>=3.1.43
+packaging>=24.1
diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py
@@ -1,2 +1,2 @@
 __author__ = 'socket.dev'
-__version__ = '1.0.7'
+__version__ = '1.0.15'
diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py
@@ -46,6 +46,8 @@
 all_new_alerts = False
 security_policy = {}
 log = logging.getLogger("socketdev")
+# log_format = "%(asctime)s %(funcName)20s()  %(message)s"
+# logging.basicConfig(format=log_format)
 log.addHandler(logging.NullHandler())
 
 socket_globs = {
@@ -396,29 +398,35 @@ def find_files(path: str, files: list = None) -> list:
         :param files: override finding the manifest files using the glob matcher
         :return:
         """
-        all_files = []
         files_provided = False
+        log.debug("Starting Find Files")
+        start_time = time.time()
         if files is not None and len(files) > 0:
             files_provided = True
         for ecosystem in socket_globs:
+            if files is None:
+                files = []
             patterns = socket_globs[ecosystem]
             for file_name in patterns:
                 pattern = patterns[file_name]["pattern"]
                 file_path = f"{path}/**/{pattern}"
+
                 if not files_provided:
-                    files = glob(file_path, recursive=True)
+                    log.debug(f"Globbing {file_path}")
+                    glob_start = time.time()
+                    test = glob(file_path, recursive=True)
+                    files = files + test
+                    glob_end = time.time()
+                    glob_total_time = glob_end - glob_start
+                    log.debug(f"Glob for pattern {file_path} took {glob_total_time:.2f} seconds")
                 else:
+                    log.debug("Files found from commit")
                     files = Core.match_supported_files(path, files)
-                for file in files:
-                    if platform.system() == "Windows":
-                        file = file.replace("\\", "/")
-                    if path not in file:
-                        file = f"{path}/{file}"
-                    found_path, file_name = file.rsplit("/", 1)
-                    details = (found_path, file_name)
-                    if details not in all_files:
-                        all_files.append(details)
-        return all_files
+        log.debug("Finished Find Files")
+        end_time = time.time()
+        total_time = end_time - start_time
+        log.info(f"Found {len(files)} in {total_time:.2f} seconds")
+        return files
 
     @staticmethod
     def create_full_scan(files: list, params: FullScanParams, workspace: str) -> FullScan:
@@ -430,7 +438,16 @@ def create_full_scan(files: list, params: FullScanParams, workspace: str) -> Ful
         :return:
         """
         send_files = []
-        for path, name in files:
+        create_full_start = time.time()
+        log.debug("Creating new full scan")
+        for file in files:
+            if platform.system() == "Windows":
+                file = file.replace("\\", "/")
+            if "/" in file:
+                path, name = file.rsplit("/", 1)
+            else:
+                path = "."
+                name = file
             full_path = f"{path}/{name}"
             if full_path.startswith(workspace):
                 key = full_path[len(workspace):]
@@ -452,6 +469,9 @@ def create_full_scan(files: list, params: FullScanParams, workspace: str) -> Ful
         results = response.json()
         full_scan = FullScan(**results)
         full_scan.sbom_artifacts = Core.get_sbom_data(full_scan.id)
+        create_full_end = time.time()
+        total_time = create_full_end - create_full_start
+        log.debug(f"New Full Scan created in {total_time:.2f} seconds")
         return full_scan
 
     @staticmethod
diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py
@@ -1,5 +1,7 @@
 import argparse
 import json
+
+import socketsecurity.core
 from socketsecurity.core import Core, __version__
 from socketsecurity.core.classes import FullScanParams, Diff, Package, Issue
 from socketsecurity.core.messages import Messages
@@ -10,7 +12,9 @@
 import sys
 import logging
 
-logging.basicConfig(level=logging.INFO)
+log_format = "%(asctime)s: %(message)s"
+logging.basicConfig(level=logging.INFO, format=log_format)
+socketsecurity.core.log.setLevel(level=logging.INFO)
 log = logging.getLogger("socketcli")
 blocking_disabled = False
 
@@ -211,7 +215,7 @@ def main_code():
     arguments = parser.parse_args()
     debug = arguments.enable_debug
     if debug:
-        logging.basicConfig(level=logging.DEBUG)
+        logging.basicConfig(level=logging.DEBUG, format=log_format)
         log.setLevel(logging.DEBUG)
         Core.enable_debug_log(logging.DEBUG)
         log.debug("Debug logging enabled")
@@ -287,7 +291,7 @@ def main_code():
         default_branch = scm.is_default_branch
 
     base_api_url = os.getenv("BASE_API_URL") or None
-    core = Core(token=api_token, request_timeout=6000, base_api_url=base_api_url)
+    core = Core(token=api_token, request_timeout=1200, base_api_url=base_api_url)
     no_change = True
     if ignore_commit_files:
         no_change = False

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`__author__ = 'socket.dev'`
`2`		`-__version__ = '1.0.7'`
	`2`	`+__version__ = '1.0.15'`