diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
index 7e9a6af23d..471826aeb8 100644
--- a/llama_cpp/llama.py
+++ b/llama_cpp/llama.py
@@ -2333,7 +2333,7 @@ def from_pretrained(
                 # download the additional file
                 hf_hub_download(
                     repo_id=repo_id,
-                    filename=matching_additional_file,
+                    filename=matching_additional_file.removeprefix(subfolder + "/"),
                     subfolder=subfolder,
                     local_dir=local_dir,
                     local_dir_use_symlinks=local_dir_use_symlinks,