GeezerVison is showing github.com — llama_cpp_for_codeshell/convert-lora-to-ggml.py at master · WisdomShell/llama_cpp_for_codeshell

Expand file tree

convert-lora-to-ggml.py

Copy path

More file actions

Latest commit

History

executable file

137 lines (111 loc) · 4.21 KB

convert-lora-to-ggml.py

Copy path

File metadata and controls

executable file

137 lines (111 loc) · 4.21 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

#!/usr/bin/env python3

from __future__ import annotations

import json

import os

import re

import struct

import sys

from typing import Any, BinaryIO, Sequence

import numpy as np

import torch

NUMPY_TYPE_TO_FTYPE: dict[str, int] = {"float32": 0, "float16": 1}

HF_SUBLAYER_TO_GGML = {

"self_attn.q_proj": "attn_q",

"self_attn.k_proj": "attn_k",

"self_attn.v_proj": "attn_v",

"self_attn.o_proj": "attn_output",

"mlp.gate_proj": "ffn_gate",

"mlp.down_proj": "ffn_down",

"mlp.up_proj": "ffn_up",

"input_layernorm": "attn_norm",

"post_attention_layernorm": "ffn_norm",

}

def translate_tensor_name(t: str) -> str:

match = re.match(r".*layers\.(\d+)\.(\w+\.\w+)\.lora_(A|B)\.weight", t)

if match:

nn = match.group(1)

sub_layer = match.group(2)

lora_type = match.group(3)

sub_layer_renamed = HF_SUBLAYER_TO_GGML.get(sub_layer)

if sub_layer_renamed is None:

print(f"Error: unrecognized sub-layer {sub_layer} in tensor {t}")

sys.exit(1)

output_string = (

f"blk.{nn}.{HF_SUBLAYER_TO_GGML[sub_layer]}.weight.lora{lora_type}"

)

return output_string

else:

print(f"Error: unrecognized tensor {t}")

sys.exit(1)

def write_file_header(fout: BinaryIO, params: dict[str, Any]) -> None:

fout.write(b"ggla"[::-1]) # magic (ggml lora)

fout.write(struct.pack("i", 1)) # file version

fout.write(struct.pack("i", params["r"]))

# https://opendelta.readthedocs.io/en/latest/modules/deltas.html says that `lora_alpha` is an int

# but some models ship a float value instead

# let's convert to int, but fail if lossless conversion is not possible

assert (

int(params["lora_alpha"]) == params["lora_alpha"]

), "cannot convert float to int losslessly"

fout.write(struct.pack("i", int(params["lora_alpha"])))

def write_tensor_header(

self, name: str, shape: Sequence[int], data_type: np.dtype[Any]

) -> None:

sname = name.encode("utf-8")

fout.write(

struct.pack(

"iii",

len(shape),

len(sname),

NUMPY_TYPE_TO_FTYPE[data_type.name],

)

fout.write(struct.pack("i" * len(shape), *shape[::-1]))

fout.write(sname)

fout.seek((fout.tell() + 31) & -32)

if len(sys.argv) != 2:

print(f"Usage: python {sys.argv[0]} <path>")

print(

"Path must contain HuggingFace PEFT LoRA files 'adapter_config.json' and 'adapter_model.bin'"

)

sys.exit(1)

input_json = os.path.join(sys.argv[1], "adapter_config.json")

input_model = os.path.join(sys.argv[1], "adapter_model.bin")

output_path = os.path.join(sys.argv[1], "ggml-adapter-model.bin")

model = torch.load(input_model, map_location="cpu")

with open(input_json, "r") as f:

params = json.load(f)

if params["peft_type"] != "LORA":

print(f"Error: unsupported adapter type {params['peft_type']}, expected LORA")

sys.exit(1)

if params["fan_in_fan_out"] is True:

print("Error: param fan_in_fan_out is not supported")

sys.exit(1)

if params["bias"] is not None and params["bias"] != "none":

print("Error: param bias is not supported")

sys.exit(1)

# TODO: these seem to be layers that have been trained but without lora.

# doesn't seem widely used but eventually should be supported

if params["modules_to_save"] is not None and len(params["modules_to_save"]) > 0:

print("Error: param modules_to_save is not supported")

sys.exit(1)

with open(output_path, "wb") as fout:

fout.truncate()

write_file_header(fout, params)

for k, v in model.items():

if k.endswith(".default.weight"):

k = k.replace(".default.weight", ".weight")

if k in ["llama_proj.weight", "llama_proj.bias"]:

continue

if k.endswith("lora_A.weight"):

if v.dtype != torch.float16 and v.dtype != torch.float32:

v = v.float()

v = v.T

else:

v = v.float()

t = v.detach().numpy()

tname = translate_tensor_name(k)

print(f"{k} => {tname} {t.shape} {t.dtype} {t.nbytes/1024/1024:.2f}MB")

write_tensor_header(fout, tname, t.shape, t.dtype)

t.tofile(fout)

print(f"Converted {input_json} and {input_model} to {output_path}")