Skip to content

Mix precision #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pingbowen23 wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Mix precision #8

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
3a3c768
lowrank tailor
pingbowen23 Mar 4, 2024
86d443d
finish fp16 + 1bit, check diff2.py train2.py
pingbowen23 Mar 12, 2024
6836149
Attn,mlp fp16+1bit
pingbowen23 Mar 14, 2024
5ea7c52
delta orthogonal
pingbowen23 Mar 16, 2024
7f2339d
orthogonal
pingbowen23 Mar 18, 2024
19c1d3e
add outlier
pingbowen23 Mar 19, 2024
46d46ca
for test
pingbowen23 Mar 20, 2024
840417c
bitdelta outlier
pingbowen23 Mar 21, 2024
7c186f4
modify
pingbowen23 Mar 21, 2024
8723564
outlier
pingbowen23 Mar 25, 2024
53a0fc7
load llava
pingbowen23 Mar 31, 2024
85ea71f
support mix and low_bit
pingbowen23 Mar 31, 2024
f93952c
support svd
pingbowen23 Apr 29, 2024
29f7485
support plot et al.
pingbowen23 May 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
outlier
  • Loading branch information
@pingbowen23
pingbowen23 committed Mar 25, 2024
commit 8723564be15671746bcba4b4dd42989d751fd4e1
16 changes: 8 additions & 8 deletions bitdelta/diff2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def copy_nonzero_values(A, B):
A[mask] = B[mask]
return A

def compress_diff(base_model, finetuned_model, finetuned_compressed_model,save_dir,layers=None):
def compress_diff(base_model, finetuned_model, finetuned_compressed_model,save_dir,args):
def compress_submodule(name, subname, module, submodule):
target_device = submodule.weight.device

Expand Down Expand Up @@ -134,11 +134,11 @@ def compress_submodule(name, subname, module, submodule):
dim , fp16_col = 1024, 64

if "self_attn" in name:
U,S,V,outlier_U,outlier_V = decomposition(delta,dim=dim,name=name)
U,S,V,outlier_U,outlier_V = decomposition(delta,dim=dim,name=name,attn_outlier=args.attn_outlier)
else:
dim , fp16_col = 1024 , 128
# delta , scaled_p = solve_orthogonal(p, f)
U,S,V,outlier_U,outlier_V = decomposition(delta,dim=dim,name=name)
U,S,V,outlier_U,outlier_V = decomposition(delta,dim=dim,name=name,mlp_outlier=args.mlp_outlier)

compressed_U, compressed_V = BinaryDiff(weight=U[:,fp16_col:]).to(f.device), BinaryDiff(weight=V[:,fp16_col:]).to(f.device)
U_mask, U_coeff, V_mask, V_coeff = compressed_U.mask, compressed_U.coeff, compressed_V.mask, compressed_V.coeff
Expand Down Expand Up @@ -237,7 +237,7 @@ def set_zero(A, B):
return A


def decomposition(masked_input_tensor,dim=None,name=None):
def decomposition(masked_input_tensor,dim=None,name=None,attn_outlier=0.1,mlp_outlier=0.1):
U , S , V = torch.svd(masked_input_tensor.to(torch.float32))

outlier_U , outlier_V = None, None
Expand All @@ -246,16 +246,16 @@ def decomposition(masked_input_tensor,dim=None,name=None):
U , S , V = U[:, :dim],S[:dim] ,V[:, :dim]

if "self_attn" in name:
outlier_U = get_outlier(U[:,64:], percent=0.2)
outlier_V = get_outlier(V[:,64:], percent=0.2)
outlier_U = get_outlier(U[:,64:], percent=attn_outlier)
outlier_V = get_outlier(V[:,64:], percent=attn_outlier)

set_zero(U[:,64:], outlier_U)
# import pdb; pdb.set_trace()
set_zero(V[:,64:], outlier_V)

else:
outlier_U = get_outlier(U[:,128:], percent=0.1)
outlier_V = get_outlier(V[:,128:], percent=0.1)
outlier_U = get_outlier(U[:,128:], percent=mlp_outlier)
outlier_V = get_outlier(V[:,128:], percent=mlp_outlier)

set_zero(U[:,128:], outlier_U)
set_zero(V[:,128:], outlier_V)
Expand Down
6 changes: 3 additions & 3 deletions bitdelta/train2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@
tokenizer = get_tokenizer(args.finetuned_model)

with torch.no_grad():
base_model = get_model(args.base_model, args.base_model_device, args.base_model_memory_map).to(torch.float32)
finetuned_model = get_model(args.finetuned_model, args.finetuned_model_device, args.finetuned_model_memory_map).to(torch.float32)
base_model = get_model(args.base_model, args.base_model_device, args.base_model_memory_map)
finetuned_model = get_model(args.finetuned_model, args.finetuned_model_device, args.finetuned_model_memory_map)

finetuned_compressed_model = get_model(args.finetuned_model, args.finetuned_compressed_model_device, args.finetuned_compressed_model_memory_map)

print(f"compressing diff...")
compress_diff(base_model, finetuned_model, finetuned_compressed_model,args.save_dir)
compress_diff(base_model, finetuned_model, finetuned_compressed_model,args.save_dir,args)

tokenizer.save_pretrained(args.save_dir)
2 changes: 2 additions & 0 deletions bitdelta/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ def parse_args():
parser.add_argument("--max_length", type=int, default=128)
parser.add_argument("--save_dir", type=str, required=True)
parser.add_argument("--train", action="store_true")
parser.add_argument("--attn_outlier", type=float,default=1e-4)
parser.add_argument("--mlp_outlier", type=float,default=1e-4)

# device management
parser.add_argument("--base_model_device", type=str, default="0")
Expand Down
13 changes: 10 additions & 3 deletions run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
MODEL_SAVE_DIR=save/uncalibrated_model
MODEL_SAVE_DIR=./../save/uncalibrated_llava

mkdir -p $MODEL_SAVE_DIR

values=(0.05 0.2 0.4 0.5 0.75)

# for value in ${values[@]}
# do
CUDA_VISIBLE_DEVICES=5,6 python \
bitdelta/train2.py \
--base_model /data/public/opensource_models/meta-llama/Llama-2-7b-hf/ \
--finetuned_model /data/public/opensource_models/WizardLM/WizardMath-7B-V1.0/ \
--base_model /home/pingbowen/models/vicuna-13b-v1.5 \
--finetuned_model /home/pingbowen/models/Llava-v1.5 \
--save_dir $MODEL_SAVE_DIR \
--batch_size 4 \
--num_steps 200 \
--save_full_model True \
--attn_outlier 0.2 \
--mlp_outlier 0.1 \
# &> test.log
# done

# /data/public/opensource_models/meta-llama/Llama-2-7b-chat-hf/