XingfenD
7856f5bf4c
按Transformer架构组件拆分： - utils.py: 激活函数和掩码工具 - layers.py: 基础层（Linear, LayerNorm） - attention.py: 多头注意力机制 - feedforward.py: 前馈网络 - block.py: Transformer编码器块 - model.py: SimpleTransformer模型 - inference.py: 推理引擎 - 删除旧的transformer.py文件
36 lines
1.0 KiB
Python
36 lines
1.0 KiB
Python
"""
|
||
基础层模块
|
||
包含Linear和LayerNorm层
|
||
"""
|
||
|
||
import numpy as np
|
||
|
||
|
||
class Linear:
|
||
"""全连接层（线性层）"""
|
||
def __init__(self, in_features, out_features):
|
||
# 使用He初始化权重
|
||
self.weight = np.random.randn(in_features, out_features) * np.sqrt(2.0 / in_features)
|
||
# 偏置初始化为0
|
||
self.bias = np.zeros(out_features)
|
||
|
||
def forward(self, x):
|
||
"""前向传播：y = xW + b"""
|
||
return x @ self.weight + self.bias
|
||
|
||
|
||
class LayerNorm:
|
||
"""层归一化（Layer Normalization）"""
|
||
def __init__(self, d_model, eps=1e-6):
|
||
# 缩放参数
|
||
self.gamma = np.ones(d_model)
|
||
# 偏移参数
|
||
self.beta = np.zeros(d_model)
|
||
# 防止除零的小常数
|
||
self.eps = eps
|
||
|
||
def forward(self, x):
|
||
"""前向传播：对最后一个维度进行归一化"""
|
||
mean = np.mean(x, axis=-1, keepdims=True)
|
||
std = np.std(x, axis=-1, keepdims=True)
|
||
return self.gamma * (x - mean) / (std + self.eps) + self.beta |