Skip to content

Mix precision #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pingbowen23 wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Mix precision #8

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
3a3c768
lowrank tailor
pingbowen23 Mar 4, 2024
86d443d
finish fp16 + 1bit, check diff2.py train2.py
pingbowen23 Mar 12, 2024
6836149
Attn,mlp fp16+1bit
pingbowen23 Mar 14, 2024
5ea7c52
delta orthogonal
pingbowen23 Mar 16, 2024
7f2339d
orthogonal
pingbowen23 Mar 18, 2024
19c1d3e
add outlier
pingbowen23 Mar 19, 2024
46d46ca
for test
pingbowen23 Mar 20, 2024
840417c
bitdelta outlier
pingbowen23 Mar 21, 2024
7c186f4
modify
pingbowen23 Mar 21, 2024
8723564
outlier
pingbowen23 Mar 25, 2024
53a0fc7
load llava
pingbowen23 Mar 31, 2024
85ea71f
support mix and low_bit
pingbowen23 Mar 31, 2024
f93952c
support svd
pingbowen23 Apr 29, 2024
29f7485
support plot et al.
pingbowen23 May 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
for test
  • Loading branch information
@pingbowen23
pingbowen23 committed Mar 20, 2024
commit 46d46ca852d757ab0838ae071686781ad54bca09
9 changes: 7 additions & 2 deletions bitdelta/diff2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
from bitdelta.binary_gemm_kernel import pack, unpack, binary_bmm
from bitdelta.utils import get_model, get_tokenizer

# 离群值抽出之后 原来位置设定成多少，如果设置成0会让分母增大
# U, V

class BinaryDiff(nn.Module):
def __init__(self, weight):
super().__init__()
Expand Down Expand Up @@ -142,9 +145,10 @@ def compress_submodule(name, subname, module, submodule):
weight_U , weight_V = (unpack(U_mask)*2-1) * U_coeff, (unpack(V_mask)*2-1) * V_coeff
U[:,fp16_col:] , V[:,fp16_col:] = weight_U.T, weight_V.T

# import pdb; pdb.set_trace()

if outlier_U is not None and outlier_V is not None:
copy_nonzero_values(U[:,fp16_col:], outlier_U) , copy_nonzero_values(V[:,fp16_col:], outlier_V)
tmp = copy_nonzero_values(U[:,fp16_col:], outlier_U) , copy_nonzero_values(V[:,fp16_col:], outlier_V)
# import pdb; pdb.set_trace()

delta = U @ torch.diag(S) @ V.t()
finetuned_model.get_submodule(f"{name}.{subname}").weight.copy_(p.to(p.dtype) + delta.to(p.dtype))
Expand Down Expand Up @@ -246,6 +250,7 @@ def decomposition(masked_input_tensor,dim=None,name=None):
outlier_V = get_outlier(V[:,64:], percent=0.2)

set_zero(U[:,64:], outlier_U)
# import pdb; pdb.set_trace()
set_zero(V[:,64:], outlier_V)

else:
Expand Down
10 changes: 2 additions & 8 deletions test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,8 @@ def copy_nonzero_values(A, B):
n = 4
A = torch.randn(n, n) # 随机生成一个n × n的张量A
B = torch.zeros(n, n) # 创建一个n × n的全零张量B

# 在B中随机设置一些非零值
indices = torch.randint(0, n, (3, 2)) # 随机选择一些位置
for i, j in indices:
B[i, j] = torch.randn(1).item() # 随机非零值

# 复制B中的非零值到A
updated_A = copy_nonzero_values(A, B)
A = A.flatten()
values , top_indices = torch.topk(A, 1, largest=True)


import pdb; pdb.set_trace()
Expand Down