+ + ## 연쇄법칙(Chain Rule) + 연쇄 법칙에 따르면 합성 함수(여러 함수가 연결된 함수)의 미분은 구성 함수 각각을 미분한 후 곱한 것과 같음
+
+ + + x에 대한 y의 미분은 **"dy/dx = dy/db * db/da * da/dx"** 로 표현할 수 있음.
+ 식에서 알수 있듯 x에 대한 y의 미분은 구성 함수 각각의 미분값을 모두 곱한 값과 같음. 즉, 합성함수의 미분은 각 함수의 국소적 미분들로 분해가 가능. --> 연쇄 법칙
+ + 위 식(dy/dx = dy/db * db/da * da/dx)은 **"dy/dx = dy/dy * dy/db * db/da * da/dx"** 으로도 표현이 가능 dy/dy은 자신에 대한 미분이라 항상 1
+ + ## 역전파 원리 도출 +**"dy/dx = dy/dy * dy/db * db/da * da/dx"** 은 합성 함수의 미분은 구성 함수들의 미분의 곱으로 분해할 수 있음을 뜻함
+**"dy/dx = (((dy/dy*dy/db)db/da)da/dx)"** 식과 같이 출력에서 입력 방향으로, 즉 보통의 계산과는 반대 방향으로 미분을 계산, 출력 y에서 입력 x 방향으로 곱하면서 순서대로 미분하면 최종적으로 dy/dx가 구해짐
+
++ +dy/db는 함수 y = C(b)의 미분, db/da = B(a)의 미분, da/dx = A(x)의 미분 +
++ + "y의 변수에 대한 미분값"이 변수 y, a, b, x에 대한 미분값이 오른쪽에서 왼쪽으로 전파됨을 알 수 있음 이것이 역전파, 전파되는 데이터는 모두 "y의 미분값" +
++ +순전파와 역전파간의 관계를 봤을때 순전파 시의 변수 a는 역전파 시의 미분 dy/da에 대응, 마찬가지로 b는 dy/db, x는 dy/dx가 대응. 함수에도 마찬가지로 함수 B는 역전파의 B'(a)에 대응 A는 A'(x)에 대응. 이렇게 변수는 '통상값'과 '미분값'이 존재하고 함수는 '통상 계산(순전파)'과 '미분값을 구하기 위한 계산(역전파)'이 존재하는 것으로 생각 가능
+ +역전파를 구할때 C'(b)는 y = C(b)의 미분값, 이 때 C'(b)계산을 위해선 b값이 필요 마찬가지로 B'(a)를 구하기 위해선 a의 값이 필요함. 즉, 역전파 시에는 순전파 시 이용한 데이터가 필요함 따라서 역전파를 구하기 위해선 먼저 순전파를 해야 함
+ + +## Code +- code + - step01.py: 수동 역전파 구현 + - step02.py: 자동 역전파 구현 - 1 + - step03.py: 자동 역전파 구현 - 2 + +## Reference +- 역전파 이론 : [Backpropagation Theory](../../Theory/03_Backpropagation/README.md) +- 역전파 구현 : [Python으로 역전파 구현](code/step03.py) \ No newline at end of file diff --git a/Python/01_Backpropagation_Theory/SimplifiedBackgroundpropagation.png b/Python/01_Backpropagation_Theory/SimplifiedBackgroundpropagation.png new file mode 100644 index 0000000..b5fb75c Binary files /dev/null and b/Python/01_Backpropagation_Theory/SimplifiedBackgroundpropagation.png differ diff --git a/Python/01_Backpropagation_Theory/code/README.md b/Python/01_Backpropagation_Theory/code/README.md new file mode 100644 index 0000000..a82c09a --- /dev/null +++ b/Python/01_Backpropagation_Theory/code/README.md @@ -0,0 +1,16 @@ +# 역전파 자동화 +변수와 함수의 관계를 설정.
+ +- 함수 관점에서 변수는 "입력"과 "출력" +- 변수 관점에서 함수는 "창조자 혹은 부모" + +
+
+
+## 역전파 계산의 흐름
+1. 함수를 가져온다
+2. 함수의 입력을 가져온다
+3. 함수의 backward 메서드를 호출한다
+
++이러한 똑같은 처리 흐름의 반복을 자동화 하는 코드(step02.py --> step03.py) \ No newline at end of file diff --git a/Python/01_Backpropagation_Theory/code/RelationShipBetVarFunc.png b/Python/01_Backpropagation_Theory/code/RelationShipBetVarFunc.png new file mode 100644 index 0000000..874949f Binary files /dev/null and b/Python/01_Backpropagation_Theory/code/RelationShipBetVarFunc.png differ diff --git a/Python/01_Backpropagation_Theory/code/step01.py b/Python/01_Backpropagation_Theory/code/step01.py new file mode 100644 index 0000000..dd03263 --- /dev/null +++ b/Python/01_Backpropagation_Theory/code/step01.py @@ -0,0 +1,64 @@ +import numpy as np + +class Variable: + def __init__(self, data): + self.data = data + self.grad = None # grad: 미분 값 + +class Function: + def __call__(self, input): + x = input.data + y = self.forward(x) # 구체적인 계산은 forward 메서드에서 함 + output = Variable(y) + self.input = input # 입력 변수를 기억 + return output + + def forward(self, x): + raise NotImplementedError() + + def backward(self, gy): + raise NotImplementedError() + +# x^2 +class Square(Function): + def forward(self, x): + return x ** 2 + def backward(self, gy): + x = self.input.data + gx = 2 * x * gy + return gx + +# e^x +class Exp(Function): + def forward(self, x): + return np.exp(x) + def backward(self, gy): + x = self.input.data + gx = np.exp(x) * gy + return gx + +# 수치미분 +def numerical_diff(f, x, eps=1e-4): + x0 = Variable(x.data - eps) + x1 = Variable(x.data + eps) + y0 = f(x0) + y1 = f(x1) + + return (y1.data - y0.data) / (2 * eps) + +A = Square() +B = Exp() +C = Square() + +x = Variable(np.array(0.5)) +a = A(x) +b = B(a) +y = C(b) + +# 역전파 계산 +y.grad = np.array(1.0) +b.grad = C.backward(y.grad) +a.grad = B.backward(b.grad) +x.grad = A.backward(a.grad) + +print(x.grad) \ No newline at end of file diff --git a/Python/01_Backpropagation_Theory/code/step02.py b/Python/01_Backpropagation_Theory/code/step02.py new file mode 100644 index 0000000..bd27c40 --- /dev/null +++ b/Python/01_Backpropagation_Theory/code/step02.py @@ -0,0 +1,89 @@ +import numpy as np + +class Variable: + def __init__(self, data): + self.data = data + self.grad = None # grad: 미분 값 + self.creator = None + + def set_creator(self, func): + self.creator = func + +class Function: + def __call__(self, input): + x = input.data + y = self.forward(x) # 구체적인 계산은 forward 메서드에서 함 + output = Variable(y) + output.set_creator(self) # 출력 변수에 창조자를 설정 + self.input = input # 입력 변수를 기억 + self.output = output # 출력도 저장 + return output + + def forward(self, x): + raise NotImplementedError() + + def backward(self, gy): + raise NotImplementedError() + +# x^2 +class Square(Function): + def forward(self, x): + return x ** 2 + def backward(self, gy): + x = self.input.data + gx = 2 * x * gy + return gx + +# e^x +class Exp(Function): + def forward(self, x): + return np.exp(x) + def backward(self, gy): + x = self.input.data + gx = np.exp(x) * gy + return gx + +# 수치미분 +def numerical_diff(f, x, eps=1e-4): + x0 = Variable(x.data - eps) + x1 = Variable(x.data + eps) + y0 = f(x0) + y1 = f(x1) + + return (y1.data - y0.data) / (2 * eps) + +A = Square() +B = Exp() +C = Square() + +x = Variable(np.array(0.5)) +a = A(x) +b = B(a) +y = C(b) + +# 계산 그래프의 노드들을 거꾸로 거슬러 올라감 +assert y.creator == C +assert y.creator.input == b +assert y.creator.input.creator == B +assert y.creator.input.creator.input == a +assert y.creator.input.creator.input.creator == A +assert y.creator.input.creator.input.creator.input == x + +# y --> b 까지의 역전파 +y.grad = np.array(1.0) + +C = y.creator # 1. 함수를 가져옴 +b = C.input # 2. 함수의 입력을 가져옴 +b.grad = C.backward(y.grad) # 3. 함수의 backward 메서드를 호출 + +# b --> a 까지의 역전파 +B = b.creator # 1. 함수를 가져옴 +a = B.input # 2. 함수의 입력을 가져옴 +a.grad = B.backward(b.grad) # 3. 함수의 backward 메서드를 호출 + +# a --> x 까지의 역전파 +A = a.creator # 1. 함수를 가져옴 +x = A.input # 2. 함수의 입력을 가져옴 +x.grad = A.backward(a.grad) # 3. 함수의 backward 메서드를 호출 + +print(x.grad) \ No newline at end of file diff --git a/Python/01_Backpropagation_Theory/code/step03.py b/Python/01_Backpropagation_Theory/code/step03.py new file mode 100644 index 0000000..e9e3d4b --- /dev/null +++ b/Python/01_Backpropagation_Theory/code/step03.py @@ -0,0 +1,74 @@ +import numpy as np + +class Variable: + def __init__(self, data): + self.data = data + self.grad = None # grad: 미분 값 + self.creator = None + + def set_creator(self, func): + self.creator = func + + def backward(self): + f = self.creator # 1. 함수를 가져옴 + if f is not None: + x = f.input # 2. 함수의 입력을 가져옴 + x.grad = f.backward(self.grad) # 3. 함수의 backward 메서드를 호출 + x.backward() # 하나 앞 변수의 backward 메서드를 호출(재귀) + +class Function: + def __call__(self, input): + x = input.data + y = self.forward(x) # 구체적인 계산은 forward 메서드에서 함 + output = Variable(y) + output.set_creator(self) # 출력 변수에 창조자를 설정 + self.input = input # 입력 변수를 기억 + self.output = output # 출력도 저장 + return output + + def forward(self, x): + raise NotImplementedError() + + def backward(self, gy): + raise NotImplementedError() + +# x^2 +class Square(Function): + def forward(self, x): + return x ** 2 + def backward(self, gy): + x = self.input.data + gx = 2 * x * gy + return gx + +# e^x +class Exp(Function): + def forward(self, x): + return np.exp(x) + def backward(self, gy): + x = self.input.data + gx = np.exp(x) * gy + return gx + +# 수치미분 +def numerical_diff(f, x, eps=1e-4): + x0 = Variable(x.data - eps) + x1 = Variable(x.data + eps) + y0 = f(x0) + y1 = f(x1) + + return (y1.data - y0.data) / (2 * eps) + +A = Square() +B = Exp() +C = Square() + +x = Variable(np.array(0.5)) +a = A(x) +b = B(a) +y = C(b) + +y.grad = np.array(1.0) +y.backward() + +print(x.grad) \ No newline at end of file diff --git a/Python/README.md b/Python/README.md new file mode 100644 index 0000000..7852927 --- /dev/null +++ b/Python/README.md @@ -0,0 +1,10 @@ +# 밑바닥부터 시작하는 딥러닝 +Python으로 Deep Learning Framework 만들어 보기 + +## 필요 소프트웨어 +- python 3.3 이상 +- 넘파이 +- MatPlotlib +- Cupy +- Pilow + diff --git a/Pytorch/2_CNN/CNNImageClassify_TransferLearning.ipynb b/Pytorch/2_CNN/CNNImageClassify_TransferLearning.ipynb new file mode 100644 index 0000000..8e48407 --- /dev/null +++ b/Pytorch/2_CNN/CNNImageClassify_TransferLearning.ipynb @@ -0,0 +1,335 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 10, + "id": "e006bc4a", + "metadata": {}, + "outputs": [], + "source": [ + "# Module Import\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torchvision import transforms, datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "912fbe35", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using PyTorch version: 1.13.0 Device: cpu\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " DEVICE = torch.device('cuda')\n", + "else:\n", + " DEVICE = torch.device('cpu')\n", + "\n", + "print('Using PyTorch version:', torch.__version__, ' Device:',DEVICE)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b20b46a1", + "metadata": {}, + "outputs": [], + "source": [ + "BATCH_SIZE = 32\n", + "EPOCHS = 10" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d236ecb2", + "metadata": {}, + "outputs": [], + "source": [ + "# 개미와 벌을 분류하기 위해 개미 이미지 데이터와 벌 이미지 데이터 불러오기(Train Set, Test Set 분리)\n", + "data_transforms = {\n", + " 'train': transforms.Compose([transforms.RandomResizedCrop(224), # - (1)\n", + " transforms.RandomHorizontalFlip(),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n", + " ]),\n", + " 'val': transforms.Compose([transforms.CenterCrop(224),\n", + " transforms.Resize(224),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n", + " ])\n", + "}\n", + "\n", + "image_datasets = {x: datasets.ImageFolder(\"data/hymenoptera_data\", data_transforms[x]) for x in ['train', 'val']}\n", + "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size = BATCH_SIZE, num_workers = 0, shuffle = True) for x in ['train', 'val']}" + ] + }, + { + "cell_type": "markdown", + "id": "2ba25482", + "metadata": {}, + "source": [ + "(1) : 해당 이미지를 224 사이즈로 변경하되 변경되는 이미지 픽셀 값은 랜덤으로 선택. 이미지 내 랜덤으로 선택해 224 사이즈로 변경" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2c4c657c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X_train: torch.Size([32, 3, 224, 224]) type: torch.FloatTensor\n", + "y_train: torch.Size([32]) type: torch.LongTensor\n" + ] + } + ], + "source": [ + "# 데이터 확인\n", + "for (X_train, y_train) in dataloaders['train']:\n", + " print('X_train:', X_train.size(), 'type:', X_train.type())\n", + " print('y_train:', y_train.size(), 'type:', y_train.type())\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e178b361", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", + "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxoAAABsCAYAAADt08QTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9eZxc11nnj7/PXWrfel/VrX23JC+yHTuJYydOnDjBEJIwDBACwwyBZGBmQpgZZgHCfIdhDYSQ+QHznWHIhAAhIQvZndjxJluybFm71JJave/dtVfd7ZzvH+dWd2uxLbXlwPhXH73qpeqqW/ee+9yzPMvneY5QSimaaKKJJppoookmmmiiiSZuIIx/6AY00UQTTTTRRBNNNNFEE689NA2NJppoookmmmiiiSaaaOKGo2loNNFEE0000UQTTTTRRBM3HE1Do4kmmmiiiSaaaKKJJpq44WgaGk000UQTTTTRRBNNNNHEDUfT0GiiiSaaaKKJJppoookmbjiahkYTTTTRRBNNNNFEE000ccPRNDSaaKKJJppoookmmmiiiRuOpqHRRBNNNNFEE0000UQTTdxwvKqGxvr16/nABz7wal7iNYum7NaOpuzWjqbs1oam3NaOpuzWjqbs1o6m7NaOpuzWjv9/lN2aDI3z58/zsz/7s2zcuJFYLEYmk+Huu+/mD//wD6nVaje6jd83TExM8L73vY9cLkcmk+Ghhx7iwoULN/QaTdmtHa9F2Z05c4Z//a//NXfddRexWAwhBBcvXrzh12nKbm1oym3teC3KDppz3VrR7HevDM1+tzY0+90rw43od9b1XvSrX/0q733ve4lGo7z//e9n9+7duK7LE088wUc/+lFOnDjBn/7pn17vaf/BUS6XuffeeykUCvzKr/wKtm3z8Y9/nHvuuYcjR47Q1tb2iq/RlN3a8VqV3YEDB/jEJz7Bzp072bFjB0eOHLnh12jKbm1oym3teK3KrjnXrR3Nfrd2NPvd2tHsd2vHDet36jpw4cIFlUql1Pbt29Xk5OQV3w8NDak/+IM/WP57cHBQ/eRP/uT1XOIfDL/1W7+lAHXw4MHlz06dOqVM01T//t//+1d8/qbs1o7XsuwWFhZUsVhUSin1O7/zOwpQw8PDN+z8TdmtDU25rR2vZdk157q1o9nv1o5mv1s7mv1u7bhR/e66DI0PfvCDClBPPvnkNR1/uUAXFhbURz7yEbV7926VTCZVOp1WDzzwgDpy5MgVv/3EJz6hdu7cqeLxuMrlcurWW29Vn/nMZ5a/LxaL6hd/8RfV4OCgikQiqqOjQ73lLW9Rhw8fXj6mUqmoU6dOqbm5uZdt6/79+9X+/fuv+Pytb32r2rRp0zXd70uhKbu147Usu9V4NSbBpuzWhqbc1o7Xsuyac12z3ynV7Hf/N8luNZr97h+m311XjsZXvvIVNm7cyF133XU9P1vGhQsX+OIXv8g73/lOfv/3f5+PfvSjHDt2jHvuuYfJycnl4/7sz/6MX/iFX2Dnzp38wR/8Ab/+67/Ovn37eOaZZ5aP+eAHP8h//+//nR/+4R/mU5/6FL/0S79EPB7n1KlTy8ccPHiQHTt28MlPfvIl2yWl5OjRo9x2221XfHf77bdz/vx5SqXSmu65gabs1o7Xquy+H2jKbm1oym3teK3KrjnXNfsdNPvd1fCPVXbfD7xWZXdD+921WiSFQkEB6qGHHrpmK+Zyy61er6sgCC45Znh4WEWjUfWxj31s+bOHHnpI7dq16yXPnc1m1Yc+9KGXPOaRRx5RgPrVX/3Vlzxubm5OAZe0oYE//uM/VoA6ffr0S57jpdCUXVN214Ib7W1pym5taMpt7Xgty645112JZr+7Opr97h+H7C5Hs9/9w/S7a04GLxaLAKTT6Wv9yRWIRqPL74MgIJ/Pk0ql2LZtG88999zyd7lcjvHxcQ4dOsT+/fuveq5cLsczzzzD5OQkvb29Vz3mTW96E0qpl21XoyLA6vY1EIvFLjlmLWjKrim7fwg0Zbc2NOW2dryWZdec665Es981+x3845Xdq43XsuxuZL+7ZupUJpMBeEUhOiklH//4x9myZQvRaJT29nY6Ojo4evQohUJh+bh/+2//LalUittvv50tW7bwoQ99iCeffPKSc/32b/82x48fZ926ddx+++382q/92ppLvcXjcQAcx7niu3q9fskxa0FTdk3Z/UOgKbu1oSm3teO1LLvmXNfsd9Dsd1fDP1bZvdp4Lcvuhva7a4p7hOjt7b2uBJDLQ0S/8Ru/oQD10z/90+qzn/2s+uY3v6m+/e1vq127dql77rnnkt+Wy2X1V3/1V+oDH/iA6urqUoD6z//5P19yzOTkpPrjP/5j9dBDD6lEIqFisZj62te+dj23pJRSKggCFY1G1c/93M9d8d1//I//UQHLVQvWiqbs1o7Xquwux6uRqNaU3drQlNva8VqVXXOua/a7Bpr97v8O2V2OZr+7dtzIfnddhsa/+Bf/QgHqqaeeuqbjLxfo3r171b333nvFcX19fVcIdDUcx1EPPvigMk1T1Wq1qx4zMzOj+vr61N13331Nbbsct91221Wz6++//361cePGNZ1zNZqyWztey7JbjVdjEmzKbm1oym3teC3LrjnXNfvd5Wj2u3/csluNZr+7PtyofnddVad++Zd/mWQyyc/8zM8wMzNzxffnz5/nD//wD1/096ZpXsEN+9znPsfExMQlny0sLFzydyQSYefOnSil8DyPIAguCSkBdHZ20tvbe0mYp1qtcvr0aebn51/23t7znvdw6NAhnn322eXPzpw5w3e/+13e+973vuzvXw5N2a0dr2XZvdpoym5taMpt7Xgty6451zX7XbPfXYl/zLJ7tfFalt2N6nfXtTP4pk2b+Mu//Et+5Ed+hB07dlyyA+JTTz3F5z73OT7wgQ+86O/f+c538rGPfYyf+qmf4q677uLYsWN85jOfYePGjZcc99a3vpXu7m7uvvtuurq6OHXqFJ/85Cd58MEHSafT5PN5+vv7ec973sPevXtJpVI8/PDDHDp0iN/7vd9bPs/Bgwe59957+dVf/VV+7dd+7SXv7ed//uf5sz/7Mx588EF+6Zd+Cdu2+f3f/326urr4yEc+cj1iuiqasls7XsuyKxQK/NEf/RHAMt/yk5/8JLlcjlwux4c//OG1CS1EU3ZrQ1Nua8drWXbNua7Z75r97kr8Y5Zds9/9I+h3awmnnD17Vv3zf/7P1fr161UkElHpdFrdfffd6o/+6I9UvV5fPu5qZbw+8pGPqJ6eHhWPx9Xdd9+tDhw4oO65555LQkR/8id/ot74xjeqtrY2FY1G1aZNm9RHP/pRVSgUlFI6ZPTRj35U7d27V6XTaZVMJtXevXvVpz71qUvaeb0l0MbGxtR73vMelclkVCqVUu985zvV0NDQWkT0omjKbu14LcpueHhYAVd9DQ4OvhJxXYKm7NaGptzWjtei7JRqznWr0ex3zX6n1D9u2TX73T98vxNK/SOoEdZEE0000UQTTTTRRBNNvKZwXTkaTTTRRBNNNNFEE0000UQT14KmodFEE0000UQTTTTRRBNN3HA0DY0mmmiiiSaaaKKJJppo4oajaWg00UQTTTTRRBNNNNFEEzccTUOjiSaaaKKJJppoookmmrjhaBoaTTTRRBNNNNFEE0000cQNR9PQaKKJJppoookmmmiiiSZuOK55Z3AlBAoQjb/D/8WLHL/6mMZxV/zeBGGCEKAs/TdC/68AbBDx8HMPCEBVQElQ3SAGQWQMRDKCX3OQNYWqpDjxWIUFITikJD/VDz8/AYaEdwuYD/T134i2sk4DvwLcBtwfCqQduAgkwst+FYgDvwlsWsO2I6lMG55cpHMQOvtM/sVP3sXEZJX84jjTU4tcGPEIfOiIwtveALu3w8nD8Pjj0NIOP/B2aEtBjwP1i+C1JUmJLojVCPoT5F1BPR9gWxG6YnHSVgumTHD284/x7PESKRucGZhcgFQG5C2wvR12pKF3226CTBZn4Rmi1Tj/69M1Loz59N8EqTY49ShM+JCzYe+bQHlw5hhU52EjsBVoAVJAtwXtPkSAiIAjCo4C3cALwK+sQXbiDQJcwAlfZaAEVMOH80oh0A2W4fkEK53UDP+XLPdLQHccJ/z8+4Tr3e5GCEFXJMX2toDZssvpUkB/ymC8LDEti3/5wQ8Ssad5+OFvEczXuDDjsbM7RrI7ge9WqNYNDp2rXXFeyzZRXsCtWaAOLW0wsQQygLt3RcjctIPzE8MszjhEskkuHFmkqydGBMWbbruNjfvuZGxhiiPHDvD4wXFac620d7Zy/OgZ3vCmPdy+eR05U1H1Pb7zzBkcVUNYFvX5EpvXd7BUKXPvW9/O5MUTfOr/HOKW2zYxdvYi73tzF6NLMHSuQEt3ku6uFtSiz18+du66Zb3nFzbil+bxpwJ2393DYFcLE6fmsRI5Bm7bRNQyGBubZGx8kp3rO5jNT1BemufZ42lq8zX23zbI0qlhjnyvxo43xXngx2/iwCPH6WjvJDbQzvRsnpZ0Cs+R1GoBJz87zOx4lW0P9jHwxn4Mp042k6YjbjIyNsax5ydZqEUZTMbYvCuO2dGP6ySoiSqxTCczT59m4OYeguI8Z4+MIowEFVkhWIqy+/5tpBIlTp+9SGtXO8OnSyQsn5ZMlJOPzOLnYty8P0M0HqOykOfYkwU23ZFk19ZB/s+/feG6ZSd+IhyvHuCHL7nqfZ2VsSO4dLwpIAoiC/Eu6OyArlboaIHWHGQyEI9BYArqfpSKG6FWk1RrDk7Nw3NNfA+UimCbMSKRKAA1p0yhWCZfglIZyiXwSui5pBy2qRb+X2dlvnFZGeMGEAPikEhAXwoGFEQcmF3Q/f8NN8Gbd2Xp62zn1v96/f2uc5MgngBLN5tAgi8h8MFzwXH0y3dANdoZtq+xvn7fYaJlI8L/FRCErzU2aC1be4mtfwPmb0LpCMyjn9/qRsbfAH0fBOe7MPnnEBiw+atwxz16PfEFtFfh+R+Fk/Pw+i/AXZ2QEmCAUfGQxxahswuKQBpoAxbRzyCuwBAgJUJ8AvV3/w6q98PO3wVjK/QBMQEXjsOR+6CwANiIzjdjv/NB/PpnkRNP6eZW0OubASTD8w+j7+uKG0crJy1ADtTx65ddVyrBro0DdLa2cO7cENOTC9y3AS4uwMHCSjMaw1oAHRHIxg2oSzxH33qPgAWpjzeAH70tR4sIGDlUogOtUz0OnAAwDFpsm7LrsqQUAj28bGAX8E/u2Ixs3cij3/w2SioGgePAY/o2+YGc4Pmy4qivu1r4lJffXw220K+qvHS8NN5fb7/76w8LZsdSKO8/IZ0ORGAgJAQuWIaNoRTuUgXTl8iah1tx8eoOnlNhijEO8BgXGKG6qi0WcAt7eb11L+l4CjsSJ9aeJdaW1LpyxCISj6JkQODqTi4sE2UIAt8HBUqaqKqPV3ZRniIIfJRU+v6UAj8gcAJkNcBzJEJJpC+QSExDIEUNpQq4skKUGDZpJAIhDAKhiGQTGHED1y+jLJNfn/gvLyurazY0Gg/kxf5uGBGrH55gZT254lwClA3EQAr9NwYQ1Z8h0IZFo5UxfTIRfifWgdiURCY6UWaa0rERLhwqEE/VaO2NEikINjoeE1M+7w7gpIIRYBq4D5gCTqEHTiZs03lgCysDawQ4hF57bgW+CFzHpuvL8EUdGYe6B7WqZKkguPv1+wnEOuanZjh5cpLHn5jkzEWP0legsgRxFzriUA/0nLmwANEM5JKCbMcmzNTtqPo4tYlTjJ+d5NQZj0QK2tsFe3e20tOxk/X7+5iaHWJyIsAwtOxiJgyfhYFuKM5DT2yc6K4uMDvx0/O8+b4UscdKJPsD/uIxPSjf3ANeEjwH+t8CkQx8+0swh55rY+FL+SvPXqHnvzJ63uxeg9wAPbvV0RNvlZVJ+EYYGaBnJyu8DqzsGQqXLpQGKwvo5d+92ohd/08yba3s3tRPdf4Cdc9iXcRiLu/Rmk7zuttvYs/uJL/9+48xdL7I5jYDHzg6W8edqhOzdL+7HLYl6OyIUpipcrAAvTFYn0qxpcPifx/N4w1s4qZbB/n0F4+xpT/B3HCRzt4krqlYynucW5qmPnoIw/RZms7TlxJs649x9OIEu3avY2uvYnb4eeZqisnFMp4vwU6wc8NmxBZBYXaKfdu2YEYMPvvVF/AlHHz6PJ1J+PqBOVKZGPlawNzZErMTZdpz1zW9LSPhuxgCSq5LzrYwVZmudVFqyuaFg0Ps2pUi6sxTmsiz0NaG5UsKs4KBbZvofn0XdtIlnnUZnZhgqWLwra+MogIfLxsQmZwmYSmMukml4NLVkiMqfISAs49NsFifJaJibL33XnbuuZnFqc+AUPTt3UFMjOKYEVKmy9zMGEtFn87eEm27s8xPjSGNANew6N2coyXaglt1OXfiDDff0ontR/DKMRwES2PzuK010v0mvlPDd2wOPz1JS2uUlnaL7hYDx62uSXbU0GOz8WoonAo9xhraSgMG2tCPgpmDdGhg9HZCRw6SSYjHwYxo3XDJ08q26zs4jkO9qqjXwHdBBhKlQODhS4lXq+D5knLFZ34JCnlwi6AajoqGkeGwYmA0DCR3VRuFbp+IQzILHSnoikObhFodgjpUPDh7HtapAnZ/cU2iK85DkIJICkwblNDrp5JaPzAMsEyQFgR2KFMPkP9ARoZx2Wv155Lvb6PSD4AzDS3/CfLFVYaGDZmfg73/CVQKvLshGYULh6HlJnCE7qMWWuixjZDbB4mOS/qvlDYMdurPFKHCAthKKzANJ6ljoHgb5BZhzz8DOaDlYQmYdsA9COUiYCB6b8O6txNv6j+hyvmVc8TRY0Lp5iOATqDAleueQvflGlc3RK4Bs5Ua1VPnSMWjLFRrGApOjcNpV4tRhJftBDa3mLiGwooIDGxcx0Gg2Jw2sAyYy0vmgF4B7d1dFJdKfInSsm8waQtubUtx6+Zt4CieOnqE405ADHhHS5SYZzBdrjFxYoSp2BTfkIoEcBiYQQ/NeeAbBcXGmOB1MZMLNZ+6hOJL9LeogJ/c30NLIsYfPzZMOTTQIwL6otDyUh7zF4Hrgx8YSEciAoUZ0U48VQ/AD/DLPtL3MKQAJVEqIFAuLnXy5ClTuOowMTEwDRPLsjBsG2FZYBgIE6yoBZZA1S9tsFIKgUBYBtIDKRUykMhAIkwDhEJ6vjaEpcIwTO19R6GkIkBhIJDSJaCGTwWJwkcBPgKTQPlYhoUM6gSeRyB8DCPDtWBtKzEvbnQ0lMyGgWFcdowChAVEQIRGRePvhtdI2eHxgfagY6IneiOMfMSBPhPVOYDI7oIgilrfw5kTBxmfWKLdlty7N8Nti3GqQ0v4SnfWfNiuXcB3gGeA/ayM4QLaAAEdxTiAHhytwH8AxtYoK7dcJbdeR29KJcXffvkAlYrk3te/mfXrF1DxadZtaePRb5f47pcrzP8N3LUeelKgAhg9A51ZiJagWlBksqP0xzpJ1DMk4nvYd2sXwrnAs48uUk5K1PQCsTeeo6Wli503ZVmcXaQsIS4hIyG1BGMXoccEv1Ig0jpKtH0dsligf4fPD23qZVSMs29I8GhRkqlB6q1QG4PSBDx/FpbCZ9kKZMNH0lizPSCtIBB68EdZMeauG2VWIhll9IR6IyIJgnBxYcXbejlWzwAv5Sp5NSEgvgYr7YMffhCzvMBf/+Vp2te1ctu+DRz53glUPMPcyBC/9JHDlFyfdCLJ8HyFZDxKzfcJVEBNam/q5QgCxdxsFS+URcWDuufw7cU6hgEPP3aOxw6PMFuULJ4p05U2yUYFhaU6sWgroqOX+cI4rbkW3vaeH+fwE9/FrQT0DcIb7tyNPz7EwkKFnsEuFqdKpDoHaM8oeruTZDs6yOwe5L//yVeZ+fYpKlWfeMTAdSWTFXDjMVxhEU1H6G9pIRW3qYu1KXzCL2OWS+Ra4Dufuci6bRHW74kxemqeA88s4v/AeqJ2hdZeg46WDKl4Gl/NU0m10tvZzdLCeWTc5s0/vpu/+dizTByoEM0Kcnvn6UpG2H5XltklyeG/H8GqTbDxdoGjoFyFRDpGdRJm5hycRB+bbn2AWvA9ykEJjCQ9G7YStaOcO5Xn/AuTBBWPaFuazs5OshGPodFpjoyN0rs5QzIumTmxyGg6QcdgCxVl0ZXsoBQFa2mGnftitCQSuIZBYalCLpVj354+FqsL9K/ftCbZUeHSSEYQ/i9ZUeQly3O6kYVYG+TaoLsDejqgoxVyGbCiBr6hqHqKkgdOoD38EoXvg1ODagWcKvgeIJXW/4SPwsd1dQQjn4fqIsgCK56PxmS1OnrRaHdj3Dc89THd1nQWutugOyHoNhWxqm6PLcFX2gN8JACjoHj7GkTnlcENZSUS4bqIbo8MFV5DgBlGV4PG9w1Zfz8V+0aEt2FkmKu+W/3Mv1+Ix0H+IFjPgfkXLAvD2A/9/x6CFh21CHqg9T9A8Tnd+fywvXG0FZf9IdgY1UI2BDgCYSt9trgBntLKv4Xuyy5ghg/GQBseznbY/B8hEdHHJIGqwpQzMPkXBDIFPfej1s/hPfdpvVhGWHmOFnrRbBi7BjqCkkIvvFeD4rIozvWh7AeUS1VMoN+GU65eanvC5ofBRnrdAN+AZAAiCJh19HA6XpIUlA7wVIC6gocfO89FR/JCeFs2cPu6JLdtX4d/cQSxVOAdnRYdkwFzAWxtb6d/3Qb+4pEn+HLZY6LsUQ+vXeTSJXpSwWxN0Wv4JC3oTFnUfclUVVJWK8u1BXQK2JyF/b0ZDg2V8ZU2MLrjcGscthsg8tcvM98FKU2EaWIoA8MUCBMMSxBUfFTdBRTKAIUCQyFVQI0qc0xRo7I8xaz2bfp4IEAKA9MQy19IKVHL/ySmaYLSBoXveRCAYdjIQBE4Hr7nYwiBQuJ5AcrVxwglIDDwPIUMQEgAQYCPwCPAQRJgkcTAJiDARCAwkGHkJMDDjEWQyryKZK7EdRkaL0WXWh3REC9yDGgvDTaICBABZYYGRwJk6HXHCk9khsaGCxigEkAcVBeIzhZUIouIJ3DGR4kN9LMjFuW2QZvvBus4ODzH629u5c8vFIn6PhvQEYsIeqy2E1rowFNAB7AdOAscQYf4Go7zbPi77wAfuB6BhTBNGFinFYlIBPA8HnnsKQr5STpbExwdGiMwfBJZwaa9cOoQfOMFeOtm2L0BqkfghAS1C9rbIVqqUcmcIlLOkhgtELEEkXv2kRmY47G/OM35Ew62muH2NwraNrWy8UyeEwWJ5ULMhf4ITJyD6TbojcK6iRmMOERS/fi1SWIRD8bi3LujjleELSWwUlBuNfj21w2+M+qTDR/RcCjLJDoCHdGPF9Bz8hJ6jkyvQW7ACk2qwqVUhleK1V64hpfqRsJmRdEyw78bM+XVjJoXgdkGyf7rv/yX//pb+K5iJu8xszSHLNU4OVYk2yLZ3NbJzdtbOTZ0gaTh4ZkgIiaZGOSikBKC2TmfgnvpOaXSXpyEBaYFxTo8PumRiIFrCIouFJe0J9wPwMwmMMserakkHS0J0rEcxdkZHn1+mF/+lbfTmdzMwSefZUtbP2nbwurvIbOuHcuMc3csy9ZtG1maHafkLlCaneXkmWnigYshDdpTERxlUai65JJx9t68Eb+wyIWxeXZs6yGKYnh0bYbG0MMFdu6D4aMwOeZQEYqeLQn6bmqjvVTm/OQCb9g/wPRsHsMI8FG0d8TxT5ykdm4EkVzAsNPMlubo2JBGJKPc/KZeegdTCC9PW8zle1+/CHFFadHlyLd1V/QtKD9SQtWhHDnB7DvuI4ONZdbZ1tWGZ0hmjowQ3dRHOuOx7uY20lGbZC5OLBYwcuocRpvJlp39nH36AkYR8qc8nj4zTMeeGNGeLEF6kYhf5479vWSMJVwPyiWXvtYEXds2IUWdqfEqrampl5HSi6AR0ZBcqmw23ltAGuwWaOmA7m7obYPONmjLQjYtiCXiKDNJWZrUag5lr0jdCwik7oNSgBtA2YViGWolkB4Ypo58GEJH5EplKC2AP4vWgBpzyWqjYnUUo9FWgebNNtYhC5IxaE9BR8xgMJUiJSWVQo2YLYnbCuXDkoRDi1B14d+sQXRRqanBvgemR6j86vYE2km6rMSrxkJraINENSJHocHxqlKprFWvq2V5vhiN4dVElwnFPvA+BL1lqH8DXBfi7we/DRaFXsgNAaoDOu6DqKW9YXEgp0AZ0Hc3rAPmhe5oDTqNQnc+nxWGhQEio3UYIkorjSYEloBoBFwwbJCuAsslrr5OdW4Ktn8cOl+Aw5+HrNIGROMaIZNj2ZBrPFMTzRkqcPU1MIy6rQVRS+D4+j67Y5CNwkhBX6aEXt+jwCwwU1lpbixsznNAbVVny4S/+0zRv8T+9YBHhsvMjp9knQUPvmkv0XgH0998jFjZRdXyiIUzRIEhVpbJhv1kcenS6QOjoQMj6vqkhW5rDlhAT0U5tLGUrsOx4+NEYinevaODbEJyx+Yo/X4Vcx6kbL9uuTkOuI5FPKYflPQkeAIcH2oBXs1FSIUwRJgqYCCERRWHAnlaoz4398BCFY7NQ002Ar8OvvJRQqGEBCNAKB+phDYylMSwhFaYpUJ6HoETENT1pKaUQtZ8At9HREyk1IqIkhIlFUZgg5RIP4yW6tbrY/BQOJhEMImG5oXW6IVtIEyFpyRmIoNh2Shxgw2NlzIyGp+/3MSm0IvBFbxOW39mRHQ0QzToLEZ4ziTgg0xCENV8VTsiMGIpZK3IuU8fI7NrnFpLG5MnpvmRtxZRtQ4uHLpIVEomWaEJg+6AnwU2oQfEk2g6VQU9eA5xaRSyAHwa+Fz4//Ui8EFWoK0DKko7XywlOXrsAtKFSj2kkimFJ6G7H2YuwjfOQiYJGzN6AV1cgFYBya4UQlpU0nMYHQnMmkeyVGDrvm1E0wke+fghzj3t02rPsu/udWzdlGB2qsyco3MrWkxYcnTORqZNEZ0u0trrY6YGsIMUXmmeNhlnyZC8vRdywxA4sP71JluMjWTLU3xroUgZHeXpRw/oWigrK3w/gXYczqHn+DVhCT3TrPY03ghc7m29UWj05wBExEBFJHYvSBOCcbSArlULMCDeJ1i/teW6m9GS0jpVr5mkMFdlfKyILcCpVZBxn6GJKSK2olANSCZM6rU6fiCRBiy6ek1djWwmQblcI2IaeL5W+rK27kdLDqRjFtlckmIlD0DUNkmbCcxUwPrtfYyNzXDw2cP0tEZBKS4c/hoXh0eZnKqyVM5x+y3dLCyCUzHpyOXo7wyIRBVLF12++63nmcLGiHdgAV1tKZ47nae3PYPjBBCUkEGdU2dnMJTg0OMniasyI/Pu5WK5JkQNE7slRrbfp21vitFSwNGLPm+4r5U3vt7Cr1RpzcDImMOR42Ns6beJZHoo5F0qRydpeWcfcStO1YcHf76LTEsXFhVKjsXI6BSTM3VyvRCMGwy7ARsH2tl7fx8Pf+841UIQDp4lWoJFMsyTj6aoS4sXvnuaWsTg5r0DRFJROn3J3PgcpdIMJ4cCKmWJwKKzN487GeAJibEerLxBd1eEpYUqslDmDe/aQHfOpppP4MdN2owyrl/n8KFTKDdKf0+EaKJzTbJbjjg2XorlMSFykGiB9k7o74TeDuhsgZY0xJMQjVooK4UrUjhBnJrvU3f1rG2Z2smsgECFzukAKg6UqnqOtSOaoaIMTVMtVyFoREJL6LFXD9u1WkFePQc0eLOJlfsxPYjZkKlDXwJahUAFNgYBNalY8OtY4U+m0RGWtaAPcJRWYAIPZFVHNlQk1N0VeFLfv1xldBCAbYa34WujIy4gEodideUWbwga7AM7jLg0ni+E/HBWLqjds98fpEKeWf1mGPgz6PwenP8bqPZCaZWRYSvt7bRt7daOKu0FE0BEIHpM8EDNE0Yqwu98AeUwmhEVywwLogI7odU004OgEdGLgJnUdDcqIB2XypyDGvyv0J+CZ/4HVAJNCQjQa1yDRtjw1JqsKC4S3cGi6DG2GjYr3tA1IJDaSOpOCLoTiqHFleWpjM6NiAG7TIvtmTYSgWSxOMccK8HA1Xgp905VwQkXjrnQPe2wfTCCFAIbOD1RoTheeVEd86W6koMeO43ltXHsEuAqqNfh6LkKCbtG3BQM2ArR140a3ENi3w523vWDL3H2q8NzIVAmpj2CsB7BNOcgEPjeTqTYASpCIMWybicMgTIMHFwS8RI/8lbFhgGo+/DEYfjzQ9o56+IQSE93AxMkAUHg6ztTNoZha3tZCqSUuPUAz3EJagHS8xBCGyCmJcCQSC9AqQAMgXR0UosIjWjDMAikQpOnHBRFPCrYxJH4CAwMTBQKYQQESMxYFMuOYJgm15rW8opyNBpYHcW42nUviXKs5liJkOq4+jMTVHwlbCwahokLdAoWpmOc/7pBRzJPuutx/PUZZC7GiUfOIRH4Ppw8sMBOFmlbkPwk8Hfh6040JeqdaOt8Hdq4KKCjFTNoD32DItVILmoYJmtFxIJCCSLtkEiDkYZgVq9n+SBcQBztyVK+XkRSrVBahKdHIboF/Cp0xPTkFpksEm3rJvBrlHosLLsXJSFeWiCQZdreFmBchPHzPu2ZaTZt7GVbWxWnJFkq6zalI1DwYCaj59sN1RrJ/EWiiT6sapqY7RFZgnwW+gd11ClZ9+jcnuffxVv5gQPwJ+fLnPQlQ+joUIIVp2UeOMNKIGKNaos+wavhmnu1FsGG20WCSkpEp04Ok4usZNJd4/0YKdi3q5NE9PrVhNnZPNlUBL/sEsVGWA6GB55rMH1ukqW6Ip5qZWBXP5NnT3LXTd3MTU5wbMJfDsJ0dGRpa4niu5JSxSFwLbq70lwcL+AEAc6qZkUsi/6OLItzRWKGQSxmkrFc8spnJj9HOm2TSsToyCbY0d/O9GSdv/7qPJsGs/z0AzczceY8m/fvojhbpTaeR7VIuntTfHV0mmgiym27drJnxxb+/W/8JYEsIoHR2TJdHVkCt8L6nh7id2Q4/tQROmSds4sOs87a3KoDd8TouWkz3duLXJyr01WNkDQlti/oG+hm4oXj5Gdh/YZenj06y/oNPUwOTRPvbmPD9gyJdZ1cGC8wPTxO+/p1pFoieHNFXM9hMT9PYcojl4aNuwS9vZ3s3LOOeJ/B3aKN50/PIxZgcJNBxp8mbkIsk8BM5VDRANMXyGqR0YkZqksBQd3FmZIsviCxu2Osf0sWJWq4JY+gCLE+k10PZOhblyFbKZNMxOjvSiCCgJLKcfHCDINdKXpaJMn2JIWZJOdOnGXT7vVrkt2y0t7wysbBykGmA7q7oD+kSLVnIZuGRFwnOLrCJu9bVGoGnlelVKwwM12jMOmQkIp2qee/iAVmCpZSUDchYoOwtWFc1+uqZpEE2vhYpkY1crsCtLIWWdXe+qr2N74L5wfhQMqDngDW2ZCKS2SxTiEvKTmK5+cCRjxBpzDZGI2Rr5dZXKNW342eIopo76ZfA+VA3dLrod9wwjWMDF/rwkH4QmgahEIrdLXqlQyDxv9ramKD0mODZYNphHTmhqEhddQlaFywoaQ3DKJXEw2+jGloSlTqXbDjXhiua+2t8VxNQi+KWKHNytBIsRWZDqjlwZXamhUChBA6X1QBaaEXbFOgbEUkB9GIQAS6HzoemBGFiOh8yNISUBMwm0S1/SyY83Dq/VAsrThcXVYMjEb0IpTnJa8Gja/BJ4qgF90UK3SuNcCX0GHAzs4k0/kKrlR02bDg6f7YYgge6B/gnlvvZueOvQwdfJLvfucrPKEURa790VroaMd6tH7w+edPc9PRs8R8iQTalR6qa7TTgaunbhbR+bYukPMk2z1IODB+rkz39nWcGJ6nePo3ecu9D1zXtdw6WPY8keSfkkzUtQPZAsc9SHGmjcLFdyDmbkFJkEGA9AN85VIiz4aeMr09ehzFDdi/HY4Mw6E5HVdo2OkKhRAKGfgYxqXrmWGaBFJqI0aIhrcaQxByLKEx+JRSOm/D81FegCEsMAQq0NEMgYukiksRDxeFQQQbgUWAwjRsDNNERC3MRBTfVRi2wrRfJeqUuMr71bicPrWaUkV438INP2wM/MakFNKolB2GI61QdiaQANGTonXLINm9KWoLJuOfP8H48TkyEbjowCyKnxiAi/OKr1UVDnryvhn4e3SU4g40hSqDroLQoZvEAvC9VffRsG8CtAHySrD5Tu2xS+cgkYT5vKaDPdgv+NoYFKUCCV6gO69X11HfSAymK1Cug6rDZAE2+2DGfRjIE2kbpO6OU0tGQCoiVpaRkxcYGVHc8wDkJuHoZ2qk7x1j3YBichqKUSj7ofNEQGEM7CpUphVZVSdlDNO1J87MaYdTh7TsN7VArgzmKfAzJaKDGfZE4/w/nR4fORwwUnVZx0qBDEIZ59FGnB/KeU141eL/NwCrB0MEPZocLkkYVzMhrSEI+3ajgtrVYAiMiIX0NEeztS9Ge5vJ2Oz198B02aErFuHcnEvMgL5Wk5PTgQ7DSkEi0cbr9+1mevYUvVnJ+PQk4ws+DpCIxXGcOi2ZJJOTS3iug8SkPZdmYr7Kz/78j/P1r3yd0fEFfF/f7EK5xjMvjCAURCOSvVta2Lje5vjFEu0dMUaOT+Ol4dab38zi2eNM5wMiUYO2mGJk+AIbBzZhlmFxbp6jFy8y8pxD5bMTDE95LFVcgvPP8dgTpwiUpLs1RiAFLekIr7t5K15xGpEvEHUdqnWHx/IOroD27Nq4BK6lk/w6+3opevPM1orMjy/itqfZvq2Voe9WGNyyjUTaRLoOpuVjWz47bt5HSyRBtT4DxhLRaBzfjHHo209zy6076UlJLjoVKo5gtmyQ3Q5vu38Lvf0buTA2w45dAoWJ8m3a+sAtTbC4WOZrn36eW9/cxeY+i6Hn6jjFCpFAUnccpp/1cX0gCc5EnQvfdOjbm8JK2gTzHvWxgLETFSpOnZv2dZNrSXH2mUn23PdG7KlJahNnMLrTjBR9ugZS3L5tN8XqGN/95gvwM2sQXoPymoZYq47i9nRCd7vOb2hNQzqpmSWBrfXD6bxkbqnOyBnF1HPgzEN1GtwlUDXImXBTGgZ7Ye8W2LBNe4u/XIcvWxCLQXsUZsKxV65r/vRy8nkjHyNAK2RJVjjwNS6dYzyWFTnTgdYANpjQb0LOBasI0/MO0wWQCag50Bnv5F07/wm9lU4Kk3/D6eLRNfW7OMuFrZYDuFUJc65eD2wDzAQ4hr4dFUYQDDR1DAVx20RKhRPIq9a0WF37osGbf8m6Gg1jwWQ5kmFaYFuanWAYK5ENGejPXKEjuKqhSDeSqhv0rlcDebR1FRFa+baF5vsOpLUwPbRVlkQrFa7Sxwl0QkFMILIC21bE0jBrSFTNRPna2BPoe8cOrxHy02ImpOJKe5drYKQVQegQNOZBTqNpe5aAmAOj/x3GHgfS0JIAf4blskMNqlTDCdao4rXacM+wopiEuUPL+a2vQLZxWxA4HtWyIisgaxk4nqQI3JLN8bq77mbHnj1EKnMcHTrIc0pRuM5LNtiK93VZ9Hak+M3jeZ72JXeho3kVNHX9OGvzAaaN0EC/yncC2BIVbDcUbTX4roKhM0V2bztDW6KLQ599jLdcJ10lCCCZ8UmnfBJpSEW14aCkJB6dQ0Q+x4IHcv4mpOsiA0ndr1KiiOH5+D6YUtvG0RjsXwdH5rRxIZU2cIWh8z4QEoWFCkvRSSG0kY/Shq8lsOMm0hYIFSaD+wFKCVBCJ4oDhiHwPR9lGsjAIFCSAA+PAi6LeFQBczmYZoTVppSptB0fiSFMU0dJPIWU1/akros6dXli9+r/X+w3V/1wNXEvTPAWdhjJCAcphp6oZBJIayXNs5P4IoVhVvBrs5itPiPzkHS0QtsLLBZht6XLqr6ANiY6gTejx+ZJ4L+gjY48mgu42iIPcxTZB4wC4y8vmpdFbCvkLMhENV1go7KZmw+olOGdgxafPu5S9XS4PAg0D94LF0lPwuIS9GVgdg7OW7DrjKItW8C/pReivVRHTmNme6ilLRK5KPlHdAK5nYN8FY580+Ut7xBs3QrnfSiNQryuo8fOFCxMgxODggNZR7J0pkJegFuDViOkNJ+HyQqImxQ7IglMo0Tn9hj/qlznw8/rSFAbK0bbPCtr9jSvgDq1VlxH5GDN5758BW8kFgr0yGooNq7+XpXDzyJcmRdiQPvN2zFaosw+cQQiULbqnB6dJLYGfXlnAk5OVrEQzLkKy40iRRVP6nF11019TJTH2XzbnZx77lEOHs8TSIhi8S8/8GEOHHiYqpXm7jdv5y/+3/9BEHhMzC5iGILP/p8vsKHTwhjsZ6Cvi7GLo4xPz+MHAdFohLbeFFUMzg5VOTPhYWcDRpZ8kjXJp/7nd2hPKERXijve0EF7tUipvkTPjm3Mz1wk29NDaqnID+zbwOT4PLvy83zh4eeYnqsyPKFVookF7YKOxSTfO3Qcp1SnJT7FmQUPXwoSiTRCVUnnrkYgf3kUxyMUBgosLRXYs28bC0vPM6x8PK/M5NmzdA50IYwqddcgms0wfmYaZSmOnTxOd2cON79A18BGLj7jkrT7+e5Th6hWAnbf3MXGvfuxs9M8/cRFguchUp3joZ/ai2HOUylWiMVzRNMmrV0xTL/G3Mw0++/pJJtrYfOmzYjic3z9j47g5BSprIHywMoYZBIRFmfrqAWolV3MdvRAlDB73mVxVtAzIDD9Kl3bBonn2ukYMFj426d5Wo4wueTRuWjx+JeG6dhiIvz6y4npqjDbINMOXV3Q2w4dbdCWg0wKUnGwIuAaMOXC+SFYmIAzj0nyp8Cb4dLoQohF4IkqzC9AZwB9SeiKwV1leKICEx7k57W+KAyQjXyofCiDIitjMRG+GjlUl1MnQ20ohl5PtkRgQwJabD0Xj03D+TrMKM16iQPTbpktG+/gbYNv5dbJe/l3X/uFNcluhBUndTJsokK3xUMrJVYdiiF9ylNXepNrjUoNL4LGFLUqz/xKrHL8LedhhIUzTFvTtJYNDQFGw9CwdBTJMHVkw/fD6MZqClCjATc6wpEvQToNltI8+RQrdKoqOkdDEdKPlDY2wogMVU1twYdKHlpbwcgGBBWhD7DR+aExoc9vaKqRERNEkwozJvADBYHAjkGgBBELlopKV7XKhg/L+yqc+5SmbvEOsHqh+49g1l8xhiRaAWkYE4oVQ61B64uyYh02no3iFSWDW1Ixvugw7+s+vVRXyznyTj7Pwa99meOPfJOFcoVCrc47NqfIzdV5qnB9JkEVOFaW7NgguMmAoxIOAnehqdYnuJIZ1kDUFNy6voPFpRJnF2tXdKGkWl5qL3FwR4DtJvzETd0sTC3y9QmHClDy4X88/AI/fscOhmI21wtlaBp840Je2CBhgB2FdFuV8rqvUl7sAlpQSlGVFSosoUoKvwoiqcdTxITWFKQMTWuyTRvDMjFsA8M2EZaJMAwMpVkvygCkj+d5YAjseARlBmGEzkA4LghdiUrz9wTC1Fp8IE2k0p4BH4eAGpIqDmUUigixMD/DxBIRlC11LnXERtgWvh8gTFCB1Hkp14BrNjRWL9kNHavx2Wqd7vKox+rjzdUfooUlwjG/PGAEemAH6ITxJHrSiIEswvAXTpO9/TZO/s1FjHqZJLpMbVXAzqjJd/IBaXSH24tepB5DW8q3oBXgC/oSGGgFuQEbGEQbJhfQ9KobAbcK81nNMzaiBtv6+xgcKOHmqySHazywDj77gubqCV8vImYY4ai7MDapy+6mozA+D0dboOV8GZE8xRkryvHHFuiVi7S8uZ2WvlZOz1v43/QR98O+D8Lw5+HoU4p9O6DDgq87cIsP7RGdnB5UwXG16MuAWtITQhd6jjwfhWoZ8hMQJGpskSeILaUI4gF7eiTbTsIZRy/MHnptr7ISBZ7iFZQ3Wysu74SvFho5GQ4rnT8aUhgaSeCl8LuGDuDqYy5ZGCTMHz+rD4oAOe2Qk6Y2CK8Xh+ZAZTvY0mnwwsgUvR05CvkqnoJ5T/Hw8yfo7kmxK2oQz/awf89GhsbOUyyW+duvfoF9u7by8DMHGB4/giHUSpBGwVK+hOHbPPTQbaxPupxyLFqjbUzMzeNaFomWdt744G2cP3yELfEyR56bIhBKz4GOQ6wzQ0d7hnQuTiKeprsrw6FD36K4sEjXwFZSdhK1NMrZY6f4+2cnKV+l1q5pCmp+jKV8iUDCZMUDIYjH49xz5+uYz08i1xiL3Pfut1E3o8xPDnPi4EVu37OZnrYUfRv3oIJ5MnETxwxIWAYD3S0U5l3GxxYZefY0FcMmkxEMzHoo08eRS7T2WeTSAZnuTXgIxkcX8IZhSSRoffs6ZvN58vlFVCSKUyuTa7NwyiXc1iiOY7Nh305aWnOMDw8xLaIYHQrLMzDrNmYQkL25n/ZEnMLQWXpvyiH9Ml09AVMFg3pJkuiNsGVnBFuVOHYwyjt/JE41f44Lz59h9kKdrBEhm4uwcLZGuVzBTsVoa1lbNOjmW6G3S+9/kQupUVYEAhMKEpZKMDkLQ4dh7JtQH2Vl7LwEAmDGh+NnIRUALuRigk1nFU/OgL+kPyPDyoLTgp7Ey+hxmUWvJw1qVCPaAaQQ1LV/kAyaVrsrBtuy0JHQPz87DyM1vTZ4gFfR6xdU+eKjn2LvT97Bzk038YmH/seaZDfEin7fMDZWM7tMtPLesJHWiobuKla9DMJ1OKRpBWHeJHY4lRragLCNMF/G1JGNUIfR55Ch/h56aQ2hdf7ACnM3Vhsbqyl2lzsf14Lp8+DsgF3RlXk4gU7WLgIRtUw9MlLoRFtTYCYUtqPpUmocalFFTQjsuCQYc6Ea1QJvl9AeJo2m9NyczCgSaYhEFZUyWFFF3ALLUdQrQkfVsiGlyy9B+a+gXAC6wf5JSA7ApgtQ+JI+ph4+CE9oF3c0AcIDo3hpoYLGmiNZqb7SYImsASlgUgfR2RCPcvvW3azr34BXd3n6uaepLs1yolhmtlhmDt0vT14oU13D8/KAZyqSzsNL7JCwp8NkMpckWKxRWvSov8Q5961v57d/8X2c/sbX+NjDF5hwLy0GuaC0Ud5vC9Yl4rQnWljML1Ku1cgGMHd0mm+5ihNokRWBbxU9Fh85jjDi130vZkjZLFWgVjeImAaWEghD4RsST0lEbIH4+v+FqLVSn99IfSxKNaigaoJaCWjRTnZDaEdMzgbhNKhQ0GBEIcCMWIhISPPxFZ7yCfwApaSOXBgCIgbSBxnqFkpp74uSUs9VgbG8n4ZEIvFQ1JEhXcrEwiKi7VphhftzSJRpYsQMhCUwpAVS4UmP5TLPL4Nr1v8udxBf7tC9/P0lEY9wIlLhhyKccJSnaVIiCF/uCmWKGKgIK9xIyyLSn8PcmmV+bpLBn/4BDv39YRbHzhJTkjF0jfNjjt47Zxs6L+ODaOrTQbTyex+aIlVlhT7aQAs6fDfKpaHlLCv5WmtBWzSD41RZMH0uLEpaWuZoSceY9BwOLcFeA96+Cf76+bDiCCxPyiqAoRJsn4eWPqgU4JAEvw4bhsucPV/hwjhMCI/OmSmiG2eoFiU1B+ZGYNcO2P6vYexP4cI5uDcL34rAF114qAatlo4E++F82IhCGGg6aiQKS4vQ7kDU1vkG3ryPnMxjGIL41hhvbRc8MqEYXtUvGq8Y+lmcWaPs1ozLO+iNPjdoIcVZoV8ovbBlWyDthRvZeej+3PCyNhbUq3UmN9Dn6wKR017DWAyyawgHFR1IJbO875+/n6Hf+i2ePTN5yfeVus/cbJXP/8WXQUBbZy/VWo1EOsn5sfMMj52/Qpnpbc9Rc+vIwMc3bI4ee44Jy+fCSBUZj3HznbtItXaybd1m5oefISar9Jo+HX2CbDrJ8HCBcjZGfybNmeNztL1uGyrVwcaBbpamzhFvSTB6doxNG9bx2PFJPn9gjBdz0G4a7GCgK8boxRrZlizzhSJCmQy2dxEz50nHFWfPla9fcMBitYyVtJC5Dqo1nxo2CTtB2XcwfAO36BJrT1NXFv3tvZSLRzFUFdMHuzNBqVLDr7qUimUmRhfIYtE30Eap5PPoF5+gUi9jx0FlFBPTo9TMSXpzrfR1pLAjMSx/iVMnlvju0BiB9Eh2dxFJRak6HvG4YPMem7MXAvKjdbpaYPbJGeZ8H9UH09UyHW2CeMzC7hWIskO2K2D0XJnJ0TobN/Vy4tlnKFz0ae8w6N0GoxdcrGkXiUnvQI7ilM/MC2ub7W6/CVozkErqijs1CbMOjM/A9DSMnoTZR8FrkKavA0vA4ToEZ0C6Bjfdk2R/qcRjw3Cu4eEtrH6Q4TUafchCT+Y+GFWQBb3mRICfid3JjIhy1PsebxrMEqtV6VEu/a1amR5fhNGqjnAv7+mnVoKTz8wd4Lc/91v82o/9Jj1dm9Yku4YvIrSjlhNbG3ZRw+F9o9CYIi0gamiFh5jOA5ERlvMjGzkYRmhsiFCnMc0wRyM8nwidp0aoACgRRjdkWC1Mam+sCH/Q2D+MQH/+igpzVLOgSlCP6IQDTxsWygKzC8wMuL4IFXNFLCJwJBjxMJk7As6kgrKgngfTsGDMhazQc/eMr5NkEIgtWsmMOGDUBYmIposJW1OolAvFKYVIgoqiDV1/CaZGwiTGvdB6K2xogdYfguzf652EawlI3As9b4WuWyHeCubn4OSvgyNBpcC+CdpvR2x+DHXi+Usj42twSAFsNKBqQNo2uff2W7jzrT+Mke5kYGADA498jf/2x7/DpO8v24QuUJbXPXyX4QCTtiBtKnbe3Msv/pt/SWliiL/7X3/L/3xmiVHvym4QFXD/rn7GDj7LX3x3mGl3xeYygA1Rk7tyndy0aSOdO7P0lUq0SIPDR07y9Lkajyg46CpK6PG7nKJlCLyWLDclW6/7PgKlA2kTYwbdmXcSUbsQ1ShCeQTGDMXgcbLdZ+jfMkfUnsOpnkUMxRg+auD6EtfVjgNTrRQ0MxUIBDKk5mEYCMMAITBsCysWQUnwfBfp+QSeB1KGSdkG0hIooZBS4rkehmFiYIAvUa4kcPXuGAKxnAniU8cJtT5t+wdYGBhmFGUZ+CiimQSRZAwME1MqpC+xhI1hXBtr4LqrTsEqD8hVjruqfWOwzCUUCr2rqQ+iplsg3EtPKEIrXVq6EhUWSF+gLJueuzbxyCcOsKHyOA/8y3sZ+jvJc0+d5ayCfzMT8NPo9SVAGwy/h6buAJxDJ4K3A5OsTLRRYAM6j+A8cC+6ElUevS5tQkc6LlXVrh3/9kffQSAT5IvDHJ86ycjsEnOTFQoXYeYsfGMGNibhdW1wYFLzfo1gJbGxBnyvBm+xIVgH82U4VIezY1AZU/glMDph8F05Ym6MjU9O4yldcaUwBMUANr8XKicN8vUo7zrvcmgh4FsSHvA1BUCE1xHouauRuLXo6UhzogyiBfzWsKqKAxgKMV/n9R0QmdDyHkBHQtxQfg66KtU832fcCCOjEYN9scVvdXw2qo9Ltuqk1ZorUDWl+3VXeHwKPZuU0QJqhPnEqu97gFYwPE078DxNpbvuplsWqlag5pR43a038d3HD1JzV9jYQQBLRb0xXcQUnLs4qm/J91jXEWF83qXFEuQ9RdwyqAeKpWIRL1AEgULhc+KMw74tbbgRyd2vS7GhRyGiSYLpI/ij88zNVsnnKxTzLrbnsRRIClRJ1cd49wN3QtrAtk08J+DbT1wgmYzRta6PT//lVzk9476okQEwP7OIWbEYaE+SSbnk5z2ECrBqRR59apLOjW1rXni7IgWqPpw7cZZ0UKCc3EzRgfL0BLLg0NXVy+xsiUzUZKR4hHOnJxk9U8eOxfHrC7T3WlRmSrS0Z6iXXXw7xshoQM6b58JjSziGpONmA9c0+d6XL2AZcP/79hLZ18qZcyfJmT4TE2XmLrq0bQVXGtRKZbzFBfqSLlarxdCsgfQCynWozzuoKJg9ArtPsHQhwJKSqGmx47YssuYzkq9SX5AUsrME0z6eFWew18I8WSXTbzE34dN9U4T9963nO588QVBfmxrR1aV5ylV0NbLZIlwYgnPPQOGw9i6rxbU9lwA4jabnxIclCbNExoabI3DhKsrJFRyMoqbgdivoWNTz6/pcGukbtIgM641BhH2Ed7/7JygtFfm7z3+a0WlJ3oXzToTT0iePXPYVNBz0GpLHRv8Pn/72LfzMO35sbTcYwmRlb6KGgVHglRkZ6VSSwHGoef4lU6NPWBXV0BQoO6LX36AR2QjzMGAlstGAFJcxHULvq1LayLRCSk9EhmypUHCG0udUoYIlQgvKh7XvVxSPge3ArISEoW8qQD/zhDYMMEBEFaYtMITCUtrZadgQy4A3KZBFqF0AThkwH9Fzcg4d+ncDmHBRkQjuepO6AUZEkbIEQREsW1CXegPJQGlDzK8JbZHOF6E2AbRBxy9Afyv0C4SdQ5V2gbkFuv4pbH4LdCSxTECN4E/8LVR6Eb0/gbntQYzW3ch4Bf/M0yuldhsVq64tL/cK/NibB7hjxwaGzsxQNbLUxieoqhlixUVaVZWEpRXiZVGjxbIWxocB3JI0+fA/vYuZqXG8qEU0Gadm1enu1jTFmYLu6w1XR0TAfesz3GL6nPzGSTIJG1l3MdC6xe19fbzzgbu4ee/dZOol5g59k2efeoGxhTLjjmJW6amgyIpx1GEJNnVm2bZtNxMzSzx+9tR130utApNTcGI0wTtvvpXA7UI4BhYmyl9HymgnJv6EZGqJeALSrYpIvAYRePSg3hy+3qv1Xy/QVHVdblYRIFCWiRHVEQwBGMIIoxMBge/i1ev4vo5NKgnCsjEMbUD4XoBbD7AsgaEUypcEFYnj6gEmkaGj3celrPfuwMQMO5JtpLAScTw8jEiUSCKG7wUoqTfvwxCaSiVuMHVKN+7ScOvlnzXQ0L2WPwsjFgqdh6FiYDSSnSKgGru8qdBDEgHSYMS0R8CXgrEvBEQ3j9P6tl3c8hP7OfLF52gbOs26mzp5/JkhFgLFGeBRYGfYrhh6T4yw5DU5tFcsx4rRYKITkTagB087OmxuohXwHazszbM2PxVs6budcr3Khce/wh0tFTLb9vDs0Ume/PYklSkYD+AYmsrUE4VpBeVARzOM8F5Gq3BgDPZtBD8B07NhjpkFVgxiWRDpCrVSQIcCkrDuJoicgEQG3Cx0vEkSqa7jDet8Pnt0iq+dr7MwrShWV/LiGlDoaHMQMnlaAbMMshJO2jFN82Je0SNgwIBzUisYuVCWHjqx62Io0/+r0OjEL2awNDq9x7J7RMQhntIT0MJSuMlT+Hszo6NBy0l7IcVqmTZiESYq6n4fTQAVzd+MroHFMtiWolJZ4Hd+5w/Ys6UHW6jlYiWrIRWkTYOokBQ9RTJqkIlrJaDo6Xuo+noyqbmX/tp1AsYvLBCkY/gVmDpwjow5QS5n4lZtSlaKZ2aW2LtrG71ZSbY8SbLqUK3C8MgEBy8W6O9u5UlziCdeyNPbkeDCoxOYhiC4vL7uZXCRzDkm/rSkf0OMbLfJqdOLZOMBmZhNwnE0/WANENSwghoqP8qmWzcRSyaZn5sjEctQ8/KMnD1L67ockXg7Uwvz9HfG6M6kubgYY/r5UVKZJFvfuomI4zJ6epISNfr7WlEUabs9ilusUxuRVGNVgiT4FySnvneO2cIslh1h07o2glKJSiWghTgHv3aStg44fWCBB360m571HXDsHK4K81Oz0DogcLNQOuXiT0IlLmgbDEgnTPr7O0nXR6htS5DKxqlVfarzVY4+U2bdzhitnqSejWGYHo995gjrX58gvsZd1RcVlMuwUICZWRgZgsXnoP4cV82/uF546KjCmA+T4zr/oyfQ88vIy/1YgTEHm4VgC4JJJPvVdnw7w1drj/Nj6/dzi/GTLDgdbL3pPWS+9h1i8+Nsae3lwXd+jAMzQ/zxo39AIK8e7fFVjS8c/W8Mtmzndt503feWQ8/DWXTF1RhaQTrJ2r3HDcRtE+XA3q4MeSvO6YmZlcCsoaMXpq2VY1ZFNCxjJWqxzFJYbXis+r9xwkbEo1H6Uhir3qNpVZbJ8o7nKjRApFyJdlw/hE7U9gMomDph0ASqOvrvpwBDoUwIlNIbIwJYoMyQvx4F6nrdpWpog8VCa6gxIGOAvQQHl6BtB37UwHME+aJiqQjdvVDN6/lf2QLfUTpBfUpC5LDmJ0d+Dtru01GXqEJV7gN7P3TktAKQMTAMhYgtIs9/CWo/AW/6AayOjUSjBrJWpF74TTh95FLueiPxfg0Yn6ry9ttaSGUDTj1ymKGDz2N6HkekzznPYdLRVkboS1suWtDQTRpNMLjUTmz4eVYXGxgE3tVtsSNRpyMJ7ZtaKJ4f4X//n2f4u0eXmPFXHJ6N28nagptbFacfP00ubnFfW5Lz+XkuSNiZSfPz7/pxBnol7sWnefZbj/ONUxM8E+jpxkQb6QtoJ+gSYEctfvSum/GtCH9/6AUmS2Xc4Po7ng0sFaGmInrfBVfruWbUROER+IPUpv8pi5lP0z9YRAhIJmHdILy+rh2ITl07Zty6Fp4uFmUgwp05TdPQNCpY1ieCQOF7Oj/D8zxMw4RAYJm6OlXgeCgpMQwDFSgCCUZgIIMARbiDOIIgrDSlkCFpKopJFJsk0VgKogZu2SUST+gSujUPpRSmbWBFbGzL1sbPNeC6y9uu1r1WGxjqsvdc/t4NvR229iqpREjvSoU/DLUgVWMl6Smnv1O2hTQCXvhynl1tz9O1Y4Db7+9m5NtDjLtVbt1g88VzLkk0RcoD3o7OyTiHNhZOoBOOfgzd2T/GSqjaYYX9EqA5uD+PTk6qojvnMHpwrQVzE3/N7Pw83/jbOVwH7PgxFgwbNaOjDfnwuhUXDDfcC0n3HQK1IsNTs9qpsntAe5tKc7pjxmxdOerQtz1KEx7FCqQtXVYy+zrY5sDsKPg5sKwRjO33syWe4Be6h6lN1zk5Izhwxsd1wigvOprRoDbb4SvrgT8GRi8EMTAKWnhxH24T2qgbR1OlUuGrP5TdyTXK7h8UL2asN3IvPKANIrk0wYKDJVyyKegajBEzBefO1fTDK0GiT3ssnEa4yER3qA70LNgovxkaGFYMMhnNcY8mrr/pZ5cKZBUUPI/Hjwy/9G0qRSTsZQulgIWSXi5ezrnoK5isBCRwwLAoRluZmS1gLjpMVUzMZCvZuEHazpPr7SHldDFxao5Fv8bwmUmmCj5DEytcl3PjJd2el5n0YzGLwVaLH/zBBzl08Dgnzg4Riwu29kUIqjXaowLlGpQX11a/uLRYpqwK9PWnyOW6mJiawpEBNa/A4ugCF58vsXt/nf3v20W+GnDTnT1E5BIjv38MbxrOLpToutWkPaUYP7YEO2I8+rcnKV8oQVrRkouSbDdwDIEZKNxqjbHzFaIDLbzxHXuoTwzhGD4yDUHExvPqeEmBm4BaJEVbrIW4l6DaVaKjB5Jpg7FnFMV5Xa0EF0oT0LMpRndvD6l0K0RnGRzopDS7hBdPI+ouRjJJan2KuROzSOWweDpALhrsfnMPC6NrK4NxrgSVCizMwYXDUHkUnaS1Vk/1VdDwTs5UtRc+qV5qQ9BLSb+BgpNKUTZbwTKw0q/j3W9/iN6LHyWb6eBNd/08U+IYW950K3d/8w3MPP5Z+vp6ed0PvYs7Oky+duYxzk4cACAqorTGulh05nBkjQywvdXD4RyswdBoOLUME6yotpOXalB5sezY68DsUhGALkx+45c/xC/86u8ylS9qFqfSVcCMRgWjiM6rMcKcyWXKs2DZs9igZl9ubAixIm2lCEttrnxnGuHrcq/Wyzl2Xg5JAzJxiEtNPU1oJhUN506F5QpY0gpp2+GObr5QmA1LrgIkJdFOF7caQdUFFEVo9Qno7ISFMvLYAs6t7ZiWzu8QFtTKUC1ppVHUFGpO6fy82hyoP4fCQ9D+s5Cx9QIbN+BMUiup5QDWC4hquq2sRhGdP47oaCESMYglFCpwcfgE6oU/B99Z2U28Ibc1GhqfOTFPzn2UnXfsYljUGFoosz4U3Sh6vDVynkGPteSq33dbsCUL83U4XVl5hFerZrYEjI06PPznhzhaglv2TLEv28u2jevY9NQQi74iiVYHu9C0rv0DUVrnHGbmPcZjARcn67RIuAm4WK3y9W99ht0L85wp15kI4KIBm3Mmygk4X9PLq4deXrfEbO574x5G5yp84ejzOIFPdxQm1jA/dWehrw1Gl3SlQOUCKJShwBQIfAx3F3OnfhTT/DTtbVWdi5HSRuniIlSrmrVTd6Be1TRDGwMVeCg/QEm9D3jjGQfSR+EjUQjTwBQ2RpjPoRTImodfdVEKTNvCd3yEBGEYmLaBLU2UlHhKxzIUdUxMbCwMIhjEsUUcKSROtYpTd0AY+IamDUoVYEUtTNvCEBC41ya463ZbXT6xqMv+bnx2OYVKKXTvsTTn0wirWSgFIgWqNfxhWYfXG0kSsgeqEzB3WrJQh2c/fZbuzRMcOVslV5dIBTen4F0CTiudCxANr/86dKRgA7qzPYlOWO7iUsZLL9ratVmpNHUnOpl8Ej2w5tG0qrXgdz85TmdXlEOTUKyBEziAc4lzO8HKXlcu6CjQ5TIEzi9AtQbbOvTGps6CrtRXXYD5oXByR5+oXAF/I7hz0OvBhS/Durc4qNYTFDfEiSdtEt0mtxUV8azir58sY0m9wV4FHd1pFFPKo+fGIA9BB3i2riAoHDAqsB/4Etqw6wnlXQ/ltsgKfe1VR8P6faVk5pda8BqVWEICdRD3kW6A0QLxKODUyaTjGCIs2V4FvwxWEjwDZBrtZnHQnasNbZGF/V8sQHIjpBI6pFpbQ2HxQmBSEnAthQIX3GsXlhkavw34gJCSk8dn6epMUbENzpz3KLsevpzEAs6cnmWwJ0vU9+jpsnndoKRMnK88vchIIbiuR9XW18HGDTlGjg3z2c9/k3y5Rl9rApsAQykCUWeqZGLHFZ3ta7DQgKNPTjM/X2DDQI4h/zRz83lGDjiksiZb97QgC1VsBW59iQ0D3WSySc4dXSLdZpDtElSKCvxF5kZn8RzJwrN13KlAD/B5iN0eweiNkEvmKE3OgAFeXVLOz1GZGqOnp5cNW0rMzkzTPRCnfqbMxXMVjKrg2YOTHGspsBi49PYL4mnBxdOSyqwuoV0ras+wU1SMXFBsuznBwSdOc36qRk+lxMUDCzhqnvX7s7T0RDh3tEhgWOQ8xYwneP2H9lK5MMPRv1+bdjs9r710hbKem14x5+cq8NB7HfVIsCqQ91ci6wowrCiGaeE7FVo6ByktzRB4ARE7heMtsojJO3b/LDd3wrp6lt5Nm7kpsY58aRbZkqKv5Q5ExiK+YS+RE39FZ48HbkDbQJadgzuWDY0f2fN+/tU/+3ecPDvMNx7/JuWLf0d+doTiyc+yltrAqQikcpBoAzssgKKKMD8Ks/lLx91acXY2z+LsNG/dvZ0vPXGQJcI9SDydc2Ar7eSIRnXOhkBHtaXUnldYRY8KF7DV5f3FVd43DJWGcdFIFG/Q0CGMqgiuNbf0SiSj0J2GFgFZgZVSKA98V8BUVe+enorrBd8IWRShPGsIhAOyrrSQ52FH5TjnKl2U6QZsbWjkFKZlEmzdhBou4B9xcXZFdRTa1nqNDJMA1AV0J/WqYA7BXADuh2D9oE5+XIfeTnvShXwNbk5DRujOHRPQlsJMpFAmmD7UCwJvsogMPgv5sl43Ghy+huNqjdSphIKRoTyJ/LMkFbQYMC61UZBH6wGN5aoX3XQfLZJOA35oMMb6lihPjZeYrMhL0qRWoy1u0WHBTNmnyxd0tpokkknOPv0sLbEWfmhXjNhwnYVAUXQh7cLtKdjaEnB+KKC1y2bHrXvYpFo5PnUKuzKPqLsMzY9rinkSah70Z22WDMFTpYAasFmE+oiA/o39jM5VOXDiNE6gsxSW1ph8m0zpQFTCMFAYmn6oDAQSpERggFB4S9sZe+YtmHf8Pemc5sNH45qN5zgQ8TU1re6CkDpHwvWreOUqMhdFKFt3Val3H1dS6TwME6xYBBODoC6RjsSv+QSODPM6lN4JHIESBtKQGKbAl4IAGSaAS0SohZpEiIgkwo7iuj51v0agJK7jEk/ZmJYepCKMsgS+jwxeZUOjgUtCqqzyZnBlxEMBoq6tf9WomlBFl84ydK6TMNELchXEGASTcP6UR72m54hyWXH6SJkJ9OZ7ncB0SQ+GXWjjYg96UJwD/hlwAPgR4LeBL6CXgAjakLglbF8UbWg0yt5eQBsos2hnXMO7vxZ8/isTV/28sf6GuugVEMC+trBSi7vCspmsQnEMNsYha+gKAw2jqVHprlSGqVHYvgVKWSjPQ9dFmPsLsH86RmpzHypTQMY8DKvCtr2CfdMRvjXkEkUn0OfQm+sQysEGoiUIamBWIFbRteZx9fy+D3gEbdyNop05LnqQ30CH5ouj0eFuwIJ8VTRyjQQrnrIiBKoG2vCn4kPPQIz73/3TxL74TZ4/ck6XB46E1VqssPxmBC2gCLrzAiKny4PGE5DJam6zKbVBeb3Ye9MuFqYnGZmeu0QcPek0C9UKbvDKtL+kDYEED5OuwS1EkiaPPXca97LEChcYL0sePzpPn+myY0sK03R57kSR7b0JTOFwLn9txJDuzh76OluIqDI9vSlUxcHMGLiOw2xZ0JYx8e0oypYUnBo9nWsbsSOH8xCHkyNL5Ha6OFM+tXKAQ0AibfGuD++hbfseDLVI1qpTcgULlTp33Jcj3VdmfMZj7PgkLX0m7dtspr7hggfxrAFpSG9NEum0sKXAm4CaDZm+ON3r0yTbWsi1ZBjsUVxMQ/7gAgvTBulNLfjREh2bYrh+QL3uU4yY1IYV9Umd0JoMyx7WktpbWzxZY2L7Au19LSwKg9kXSrieIpqNs/W2bqZPzzF8Lk8mG2XyYI0gppBBlI4ui5bBtWktC3mozEN5AoIxdB9/FTCLjjhHQrfpalqgHYuR6+hjZvgkS7Mjy98YwkYQQQho7+xl+/ZBElMTmJkUbWqAuruIFwS07kog2gXr9u6h+pxNtlOXlowkBbds2sxXDwh8pcjaXWzZ2MvOfet44E13k5/9MM8//lWeOfDMmu5p4yZo6zWJZ+NI28CzFLFKmbJUzC69QoGF8JTib77wdd5y9x1844mDy5/X6/plAOaCpuK29EEkC/h6l3ErzCcTsGwkrN5wV6lL18hlJ2T4m8bO7st/GyuGhWmuGBtrQsqEmCSxwULakkibwErA4gs+zF1AjU3B1vXaIEn1aqXDAwyBjKJ1jjkFcyVIJylaEzjV56C0BdJ7QOWgatJ6u6Jy3qAay8JsBdlmgW1g9UPF0/u3mIsKddFDjQH+70LxGKR/CaK7ICE0BzsKnPBgsgwb09BjrBgMljbypNQGUX1eV9zEjusdf2WL5rYpqa2QeFFvYrJGamKHAQsS4rPOchGcSbTOE4SiCdC6Vgw9pE3TYH/CZE8S/unb76Z3/x3c+s2/ovPvLvCF2koOrNV4Cfg39++kL1Lm0MMXWH9rOwM7t5Ht7OP080c5cfA0mWySD73vdg6dHeP8WIHCfJ1zJYc+umjrqJLbuJXBffu5M+pyEz24tXm6Js+Tbi0wna8w+nyZs0OKL814zCjNqJDougAbBPhRm5gwGR4dBxOypt64uF6HzBrWWNOAWg1QAsM0sWwT0xer6H+KIJBIz6M+v4Mzzx5h2+2jy4a2aek+Y9f1hs3Vih5DJia+rCN9SaAEyhAIw0QIgfQVXhAQBAECsGwLJCjfI3A9vJpDEIAVMVFBGA0xBMrT++r4PvhKIpBIagR4KAIsbGwSiEgEVwVUvSpYAosIKIFSev8NXbo3ipIK3/NCI+Xlcc3iXe0xakTprhate7F5YvlzH2iUBTRDg8NDU6sUEAeRQRNVK2BWYWcv1ApwYQGG9JrNdhPOKviyXCmL3ofOB+hGh/Zi6LKw54D3AP87bMIUcA/a0Bhgpax3Ea331dAlc9eHn9XD11qpU2uFBDo6DG5TksIinBVwSum2lCUcr+jciW60fMtoOZhoy/jw16Ddg/37gUHIH4HMFhPyi7CUJlJLo4oTyEgUkTB44BaPs3mDg3MSEziEZvbE0EbOIpAJwLoIlS5IFEPHvqvbVEQbJ1Ph8QVWWHH93yeZvaqGRhTd2Xz0DTauVdR/G6amts0ueNRLZ7jrjW2M1M4xXwQZEliXUw/aWOGAZMFu0ee2BcSiq5IlRZhEeZ3wqlPk5+auEIWvfIShli2/uC1I2jD/IrUKV/NwQTv8br9lGxuMIt94YQplWigU97z5LTz0Q+/mN//r71KrXukNHxpZgLTFzlbJCwsO0wWfulR45rUptG0dGQxDMT06SSQimZkv4SuTbDpBsVzktt29qJrD6GIJM2qwsauFnVvXX9O5r4Clo0o9N6dxAp+q42L0gJqFqdkILdu7GX/mOHfu62Rh+CyRjj4mX1iiNFci2irxai6imiC9uYeJ0gyWWMTOmrzufTkOPrfE5OQi9qSBt6SozLrENgpk1qOcX+Tgw8/wRN0gl3a47/42Hv7SLG7CJpUVlDIS36szeriGnJUsTsPNd7YQq9Xo2wJD36sjLIu7friFqIhjWAluu2crI+Pj1M+P49XreGVFahB8t87IiQqVSoDpV7G7LYIFH2/8Irk9cdbtWdtsN/l34I+AXOAV1fV/OQRoj2smfL+62IRTLjBTbvhVV/p13Z0jl1jHUnWMbz/7d2yzf4LdSZ38GE9swil8h8XzedpfF8eKCjq2b+JUNINhb0FYcURU8I67tvL1b1g8PedxZuowVbdOS2+SbLtJsqeHrh0/zX3vX1syeCID2VyKVC5DHZ9itUxQV8QEtMag5IWF6aJ6HPq+pl0Evn5dCxRw4NQwzw5PXJH30ZjO7ACCRVjM6/yfRBukM1xaYYoVI0OtnjqUTvaWqyhUhgj1BGPlIoYRrlMifG9earRcNwwDAgu3qMj0CyJKsyaE76LiDtjDUP9jeOEYZN4Pm38Wkh063FIQer+LmgtLjwJ3M+FN4OU/DdQgfysUfhGcrSz2REltBHM8TyDO45+Yx0r7BJMCz5zlpo0lehInEbumWRpYx8XZClNnbiFwboW+JOZusLsU9Wd8eL4CtyRhu60rVOl8XIQFVhW8GhAIXTxHKAzTRk28H2XOgHoG5IQOeXllcII1b6kdM2BIaseqBGqWQR0BfkC3CYmoTXcsilmrMFpT3NLXy5vvfT2btm1k8ZmHEefO45aH6Lgwz/1JXUjgr0rQlzVYH7cZnnVIKfjqt0/SIiVFFwoHFtg2d5bWAY/2SJqNu/dQj3QQT9zEzet3s2+zoJCf4Ojx40ws+OyIWNy2cwdBfoGhQ98iNtBF37oexGwMuzfJ5vfeQTJ+mPLwYW5RAU9ZupCXp7RDwlaww44gyzX6WiMs1QX7+qMUag4TnmIgd/2OlUIVJmahIhOYpoVhGwhhIkWANNARdk/hBoqaFzBxboBI2zi96yQy0F3Pq0OlFDIXHPCxsTGxTIGdrmImxjFsC8NoQdFO4EaQvoPnuRimoXOcPB+/VsOtOriuC1gI3yLwJEKaaMUDCBS+9PHCnUzVch0xA4M42FFEPEpQr4BtYEcj4eKvEKaHDCS2HcO0BKYZzjnXqGtd9z4aqw2Oy2lUrPr7srnnUnhosziuW6B8EJXwfTjQVExfzJB6h+xoFPYpyM3DcR++GWgD4m7gKXTZ2kngy8DzwA+g8wIW0aG+GTQHdhTtcU+E37WiqVQNelBX+P9i+PtWVqhNLxYSvFGwuHTzVAFcnJOYNqgobLKgKwKHSzAf0gXm0QtuI3nQQos1AmQD8F6AE7OwuQ8KfTDtBES9WeLPz6GcOKLkQ7rCUqdJpjPgx7YZPLUkWfQ1herrwEOsbBVRBaJlnTAXdzWv1wbahZZ7Cv1ojfB4Gb6/Ycngl3euxvzQENzloRPBirZ8eUd8KaOkERpafb4I2mrKoI0EFy3sOnrvFwWRFJS9gAPPPUY8HkNakEhBPAPZKFxc0FRfEqzUyfQ17aWrRycSZnPglvTkmLR1Qub14tTZWbpNg45UhDmpSKWTlAslqo6LCBTb2wXzjiDwwH8JS+byIFE2GePBmzfwpb/5LkUPlOcwdPoMn7o4wkd/+V/xprfdyde/+MhV5TpU8pl9zqckIWGDE3jMVq6MZtgmtKVs4kpRrfj4EgzXoVYvImIGs4sS0zSIJWySUUVPR5K2lM9o2WOgNc7IRJE9b+xjcnJt5Y3WbTNJt7ayYV8LB781rB0i02DUDVRnnEXXZ+H8KM7uHpLZDsZOHmfmQh4zKVk4AUIY3PfhvSQHNrP+jlEWz3+Xqi0YL5apFyT+aXeZJykimo7ozngsKChOePg2bN6epbS7i2pinm17elk6PY+oBCxehPqsJD0Qp75YZ+xEnvwYrH/degZuL5Hqi1CVHl3rAlpb61RnT1NfKtKbMWh/70aOPDLJ7Ok6I9MllOXT1h1jYH2c1rYkRx5b4OTRKapmnF139K1Jdu4hdIfxWanv/wqRyrRSKRdRq3ah9dBzuULb7NeiZyskvu9giSgXCs9y+NQgmzfvwGg1aV33g5gX0xx5pkos4jL4gSjxrnZG6r20Lu2mK2YjFGzc0sH7bo0y9LDH0bmnOXrsed50690QF5gpgRoWxGKRl2/MVTA2CSW3RCJZBSOg5knydZA12NSjedxz05DaCNEsVIuQn9UKermgqReGQPdXGToohPa8RmKh48LXuYBcpapYg3FaR697aQksQbGodfH2QZbnW6NBfwqfga6WsxLtWHZICpb3ylJhxMJghSYlhDZgGoaGerH5+OUQcSGI4C9YLNUg3g2mC8o0we6B1g6IzIAahoXfgOJXoPuXYdODOgluQUJFgdsGoydwVA24HU0K7tRehsWA4KsdFHZ2hNVT2lCqm9hgjHT1z9nu/io7ogGnjwU4DqzbCL3rLZ71b2J48Q7ULX3IXQbO4w48twS3tcJOGzsW7mTvA4ug6uAl0CEgQ+l9mZI+cuFpcP4c0sc0R0iiOc5l9KK7xqjXbiuCm83Q051iVy7HA9l28kGcarFKJpkgumEjk77D0MHHmD1xktsisGl4gqNPPst3pqYYrdf4EUtvD7BtJ+xcgPd7IOKCaF8Ls/MzXPQVm/yA9XHBRWGzd9N6tt5yO62bd9HV1UXEmKU4eZjhg48wdnqOgXSEytI80ek8R2qKQEBy+M+Z8hSPSkg+O89e6wS2D3dntjHwz99LR/Q+bhr+FfIPDzEktM5XCe+xIiCPQXsuweLsDOvbcphWAleUiVplDs9dfz7f+QkYmoZkLIeSptZ1LEEQoKMISndoQ9UJ7Dn8IM+ZIRMrLknGtdNRKSgWdHW3igcCSWd8idbeg8QHF4jmqliRAIMYylmPX9qDp3YBCT1eXZ+gWsev13BqDo7jY0VABFJvleBJAgTC13q2/otw/4yGpWBjEgXbxMPV5W1jEcxIBL/mEfg+wlSYloEVMxGGLp+rpEReoxf0usrbrnY4vJiR8VK/Wf05MrTgWeF6Sl+X9VQ6hUFXt0iDiAObwFiAgRfAnNCUkkl0CH0a+Gt0WdoedPLx/0YryJ8G/gXwWWBj+F1Io+SEvgw+WkFeCr/rDM/5bHjOHJqbuJZybteD1V09gvZux5Wm3SwE0J2Bvs2QmIDhGTjhrJSJz6PlHG5BQgJo8aA8BufnIZjWEQjVD9+YVnRvUGxtr7BxBJIjBhE7oJ5T9PYr3tMK/21Wt+Ep9Hz2AFq/Du1jRE3rSrGarny1OSyTO8ZyqsHyfkxxdHTkFaNhiTUgwpuVvPh2omEoWsdQWemUl7vqG7BZ2Xl19Vpshr+NsVI830A/tLo+p+dqQ1l6kE11sXFzPxcWDhCrQ3uPRaHgk4jrhC/fBdXLMoe9FkDJ0dEM6erkTMvXi6+1Br3FME1k1OK+d93O+35gE888dZbf+p+HecvrtxA3CnQm4NSpPOcWFMPzL+4KC0Jx3b25kyUf5ufyfPzPv0k9UJfYEvV6nT/4+B+xbvMAL2XBFUKZlz0wTRNnlfJoC9i3LklKefS32CRzrSwulHn2TJ6pokN3CsoIPAXZdJyN6xLEDM0dn52q4hEH2yLRq3j40ZOY1aulI7482roMNt7ZzvRCjcK0h5oGPIgOmqRaXRbmztO5swMVlJmYXmRmPiDWZmCUZagIGhx58ggDMmD+yClKSwFGe5Rq3UCOckm/Uq5+UYbZ5yDeC+mEycBtG5m7WKc0Kxk9MEll3qc2jc62NKH9jjSm4zJ/LGDr/j669q1n4HYJhSmm8nV810WVC5x4vsrFfIa+Hf1MXVwg1+ZSyEmqI4vgQ2tvml03bcKySpT21Tn8ZI2Tj1Qo1tfmVjHWg5wM7/EGGBkArlu/xMhooI6eZypcPvxfvP+V3BksEcGRVc7NLTDRC27SI7m9l813vY/qrMfzBwJymzwSd8YYtW4jW7yJO6MChcKKxxnMmmxPwjOFJT7+v36b/vV/yoY9XRjrwWoxqY6u7T4fGYPspKTNliRtTVcKBDg+SDNU0A0w4jrfq70Vct2wNAvZDh1FMCN6PynpayqGYUJxBoI6RBJ6B3U3LMySykK5pJ1GoNcMI5SrDP+OAn2B3oNp1oOeHSs0J1iVh7FK3I33KpxrL6dYNahTjb9N4wbkaEgP3AJUUigRpTqi9LlqARRKIJO6eoVA3533HIx9AOb+CfT+P2C16soswUEIDqNrV1bRWZ/fQAiH3r4HmZy6E/VcFAZbYXsrjLow/Dfk7N9jcJvLk49bvHBUy31qErrX+ZQuPo9y/xXMfRn1ZDt8bwF2dEN/BOFA4IVRi5rSHTmB5hpFFdhgJSTK+wbB1M9BeUpvtmCwEiHy0evIGqlTb3jve2nZfBupXBctsTqZaByjXCKYn+GZA0/Tk4UdZgrlehSA48OTHBmeXFbiNwEP+9pJ+9wpXVzAtkzqKsHCUpw7egdQ05NUXI8TgWL94HoG7383A+s66cgYCHeC4a9/ncOPPstXyh4VqW+pD73UjgMvKDjn6DXnBNCj4LynC/uYhy8Q/f1/RTzRisjP0WVCp7+yVwbo4l+7+7roHhzk6fxFPFHj7GSeRU/hoF7U3/hS+NILUKjBjoStcyEMAwMDqQQGikBIrNRZEp0P054dYZPh4DoKx4NqSHd3A51zWy5BuQjro5Kbdy7S0b9IIgPphN6w1zI9PP8YZvU49aV1OHPvAX89yqsh6w6e5+H7flhPCpSU2viXEPgB+AaBp7tLgINHkYCa3i8DM/QGSHxfgglmPIoZjSwnmBumoZPLfR8lNSNCCOPV2UfjcqNBXPb95Z+/7JzhAiVQaRBpnZ+hGp9LUG1oV71AhxazNrbyGHDBntGUoefRytBWdC4GwA/rn3MT8F20Er4QnqqdFZp9Aa0An0AbIe3osdsSvsbQhsmbgO3h398PCGBPB3TGoWtzimhOkbpYxy4GtISzfyB07tjzSrNvQMvOC19l9HzlKshXoTQWbrC+HRZm4NAsJH8YErdD34JktA7zCRh4fcADVcHwU/CctDmV9/i6VEwA70JHJnKslO+uKc2T7gA+CvwK2lhrlPaWwGa0wfaKhdJwk63+LOClFRqTlQpRjd+sfq0+n4WenSRX0j4SaIspiu44jbas0tGlAmUI4nHBbGmc8rkJcklIdsFcPdCGpArtJUOHTkmF5zEhaupy8MrTpSaFBxET2rPXH9Y1pKQufQ587yBzxw+yVHaJKRN/YYpT0yUeWQhIpZJEbJsXi7k3RCSB81NF7nnL/aQ7Ujz9+OMcPzOOZRm0JGwKRc3tLZcqXDw/Q7pnkNLkxZdtY6G+ojxGhBZ9vuTwxns3kEwk6O/McnH4IkdG8rxzIM4XT9bwCMgmTDoTirnReXr7W5Guj+VZuAsVziswMxEIAloSa9Naho9D+6Yakxdn8QVEWiCdi7D7B7eRabM4+9ww+c5WlmoFHFFjdqZCOiaYnYCyA7v39LHkLHH6u0fIxG2sLOB65F/Q+tCLQTpQGYaqGfCscZ7qdJl6HoJBX0/ofqgR+lC/sEDHthjJwCDbblObmWCmrujrThIomL5YI7azlc6b+jnzuRmefmYIO2KRSEfo29hCOV/AEiYbd7SRL5SIt9lYLTbrBiIII86mLYNrkl1fPc1Y5cYlZliGqS34F8EcVzMrGgXJrwbFxvW7iJg57tv8Q+y77wE8M0p9TmHaFne80+LAlzxOP+yxLR8hFryOINqrryINnKrP+FyA9CBnZUk6rXzsP/wX9u19Pe/9ibfRuy1Hfsony/V7BxbQDqWxADJ1aC/rjcqEAN8EImBLMOsrY9OOQcc6vXYaZuhsMlciBL6ETA6mh2BiJFzHDejeAN2DcOZZqIcUWBtNFz4XSjAhoM0SVD1Fu4JTF3Wxo56tkG7TbW4YB8ub7zWkfPn7xlBUlxoUy5v8GSvGy5qwqRXsACMqdCgYAxyFqSz8IAvFzVDZDsYxVszSOv2Rs1Tmv8FS4m16sQym0VqFhzYydqCz8i9SKT2hqQXqTozSRSJzR/DyF6D0JTI75zlwAEaH9T4lltAMZWXpSJRwzqCOfQb8t0HPgN4CeqKOsgyUFdHrWISVyHyDfwYEJWBuACq/BCoCak7fh3oU1OJKdH2NuHnn7bRv2EayfT3FifNULhyjeOwxHn/ySSbnC7Q8EWFGCg7XHE6jd7C/C11dch1avzqPXkVmgXhnO+u37Ce3rhcSLXQm4mw68zyf+c63GPVc1k+Ok/7OF1iqlrlQLDNcrHKhHvDtkObUil4WC+iopYwI+gT6eaINmxzasHGAsfMeR/73caIbIsR7TbruSfFQYLG3qvgfzxdYcsGOCBYziphdITB8TNOj4Gp6lYnOz79enC5pv2PerenIhWloRV8qfOURiRyhpf9vaG0vIwzwpK4sVSjqKKEbhJ/VIF/QKsbGdYpkG8QSKxUnbVsb4zEF0aiiZI/i+J/Bm/kAfq0Vt+ZSr3t4boBtR7AsdBlay8AIzDCSqWtVBXg4FPCWa53GsbH15vVuHd/wiSTjWBEDwxKYqQhWVGAYAksoPMfH9z2UEkSjEexrpFtcN3Xqas+jMacEq4672jFXjYbUQoUrqiMXymCZB6/C1ok4SMvQ4cIcmIPQU4VfL+lzfBldQ70Pbf22ozvgAeCHgM+gO2cEPTgaTvE29AA5g46O7EPndCTRlnIZ3fGL4TlzLyWgG4QEsNGCO7qh7MPOvdvY3aeo3TXHYk0wPblExa8Snw8wXdgv4LDUg301FFoWU+iBVJP6vpLDcNt+6Heh9h242KWdMmcLYJTBvx12/ZTiX+4UVGMxHnssxm88VuSQr/NffgSd22KhPYmClQ2mfhRd4vb/ZaVP3A38E+DNr1QwV6NFNQyCF3NHNFbPIHxFw/99Lg0fNSIcyfBclUtPs7wZUktUb7ldXvW7hssfvaD2rttMZ1ecufNHcTxFNKE9iKWiIh7XG/AFaENC+XrdEBHtlRcBdGezOHWHQr4OAURtg5a2juuTFTrpMxKNYPqCk0M1AgHxIGBkpMpSMcCKGMxUqtS9l/flGAh61u3gR3/+I7jFMU4/d5CkLah7kqDm8bYMfKMY7ipfKUKleOU5hCAXNSm5Ad5V9shwlV4r13W30NNq4dkOrqzwvx6eZKaimDlXo68rRl9rmvHpAucmq2SyERZrDu3xLIulBVpbopiOZOP6TiqLkvnp8hXXuRZsuK0T04oTtWw6uyNs2pbD6LToWt9CrVTCDRzm5vPMFqMYkSwdu4oID0bni+RuUbxwbgwRKDr740yOV6kXINaqKI+oa+K0qgCWzhUxTB3ddYfUits5RHUuoLoe7nh7O6lslmRPCwtPDvHMgQUSGwTF8wGDe3aSzZR4x09tplTxMKMWtixRXIzwxY/PE0iXo+sLJDG44/515CKS6SBAxRWFxbW5VcbHrt/IMISBicBTlw9wCJREqqtrAY0g5ZVm+EtHskr1Avff+YMslcrE4ynOPauY/4aDsCV3PxRl71ttXviKT+2kZL/YT7qvC7+qUI5iYbbG9057XKjBnq338ba7fpqvPvnX/Kev/SwHn30HH/3hX8GI9TLI2iqeNVq/ANQVZJRet4wAXE87dqwq2D54VjhvEOZBCK2Q2LZONEXqcqsTJ2FmOvRn2NC7Gfo36mMSCV2pzEBfJ4V+XwOmFNy2tYvB7k6+/MwQuXKN+hQMLcLWWyDTF6ZGBHqOsyx9bXhxw6NhlFxenWr1a03oNjFMExHTMpEo4qZA1C0qbV34Ywomfw82/VOY/TwU/h68aWYrQ3jyd6EUB7EDrSH8LTpjoQzGfjCqKP9x8oUSMA/W01jyEO6xw4ggoGYoTpb1tGdIbeSZGV2WuDiiFcZcIqDW9gJ18QBkPB3uqDiQTkOuQydK1Flxw4fOJ2oKPIEaGYDaW6B+EmIj4F3Q9LeibtIrqbby7U/8/2jryOGZAd70BPnZac79f8z9d5xcV3bfi373CZVj54QGGjkDBEgwDDmcnEcjjUbSSLLl8Gz52s/X9nu+711f+9rv2df2u8G2bEv3aWzp2iNbWROkGWkiyRlmkARAEBndaHSOVV05nLzvH/sUqgGCJNAkfb0+n/p0V9WpqnP22XvtFX7rt3yfpwj7TjQdYsCgJshGDU4a0N9wqYYJmMsoFEgvUO5JsfPUx+mPjTPSmyAxlKJnz152f+pj/GijxO7CPDcXV/iHb0yRoWsXrqIIeDpb9BLd9gI7NNjbo9HvCi6WfRxP0kTtFyMGxGKCRFqjUoLVWsCQHhBrOBzsgwM6lKLwwLBB1lpn7cIGcStGqdrmoAmHcpDSwLtzz78HkXSzf9otDucAiY8vS8R6v83QaIOIoWr19UBlGmOmamBv1RVDX6Wq2KfyPZDIKVSEJ1XJUNtWTTQTibBnmqEyk8nMGqX69/Gtn8WRAsv1kbaPGQFQ2Qz9VuGTDgR4uLi0cHHQMBBomKSIkEKYOm1sCDzVxwX10CM6QguQMsCzPTzHwXVByADpOmjJ+D2N1X3X2m92GDZLcMffzjH35O+0wtoPLTS8Ot8xqX4wiII2JJHNEE7lgDCVPfgFVEH391HOQib8+wxq8v4zlC26gnI20nQpWzfoNueromoyJCqGsRs1ia6EjydQxvT7ITFuse4RAw70QTKtGFzmr71BfNFkIOaz72SOkcdHOfRonOXrK0z9qMjNVzwOoRymIrfbzy6K0vdhVJYhBvRYINfg+AFwN6D5KszGwDYgn4HKJZjLQ+8XJDg1PjWgcyqW4KUrFn+4FPAHrhqXk3QLvCVqIpnATwJ/jBrbR4BPonTh00Jlhd5zeTsnI0IX3tSpubgzK9I5ccGteok3Ke0UapJYgbqYzZ515/0G6J5g9+6HyOZ9SguXsds+6TQUy+p7W3XVS0PTVS1HgII0tKWKFLYE+HWbeMJDZiOYTkAqEbC6/mbD/Z1k30SM9VWHxXZA1ZVkdJCepF22CXQV8fC8QPF2G1FSqTjlSgXoRoh1PcLxHXluLhSo1Rf59X/+92gsL7F/GPSczvmCRyod49VqizhqvgV3qUqNRXT6cgl6ExFwNS4sFcLv1zEF9Ed1lloO0ajJJz5yjMrqNEEqxW98/RIrdfV9ZjzOJ04NcWNyleWaixE1OHJ0nO07Rrjw8hssWx5ZXBK6xs2ZZUp1G9feGrPWEx8codGyGdr9EO1anWgqTrXdYmVlnnbLItGnUZlqMn2zgq+lidk6R/b2sn2/Q9Vps74c4G4IZs60VG+CAFrz95ec95ubpmHHSItAdMzAuulh16E15xL7wABB1MNvNtEdj+kzJVIVwcSxUZquj7O2xsHxYax8hEbgI4MM1apDrN+gteZz7NBBckYBvely8OQRpHeOC6/UuDC/hZ2Xe/KjbhNdaCTNCA3n7rgPKeXbAhsk91afsVlWVqb53W/9c7LaDsbaDxHVdrG00sJMSPJDOXbuiCDSGv6w4FF9F8tNyY3nPQ5mBadPr/DUqoelRTm65+PE0kPMtpfRpE1sbYbvf+V/40LK5Wv/8mv3eVZvlo4xFUXpbl2qSHmroWxVaXQzGZrWhSD5PizdhPIK+A0oVELUZwJ2HIb+4TCDoKvax84YBuFv5VBGXhJ4+toaf+fIPv7qL3ye//Abf0RyKMezhRoXz/gciUGuT/UCiEYhYnZx53dCq+DuzsbdHluShiBAQlvgR6Wixm9K5JQgOGdDcQ3cNaAftL8AqS9B/Hdwan+E2q3+HchfBLEdFSuPocJjcyBfRJncPvADNE3DrVpkkgJNk1TrUCur0/BRg2lbYEjVD6XWUFBYr/wyGJfVKMdjkO+FXE5FV20BWakijZ1MuSchkMjpAqxfh9xrMPqbUJ8Ka0pQxaTvktLxxZWrmCtqu1xH1bAWCJ14oYz5sVyUAd3g53cN4OYTNNejnJnbYLowz/ZAMo2qifigmaU3N0I2l6XSusb6j6Zw1h5l90/+Il/87M/x67/yT1n0AvSIQeB6rMtuO5AONHsNtaUOoYKlM5bkxLrPiX05Du/O8s1Ly2zUXdJAvwfFhqTW8BjBI4+y1zyU0zcyZlBa8JmbcTkQddme1dANaKRhvgHlQLVJ2Xf/sTxA3apsJE4gA3xPYZ0Dz0NoN8kPFTHMEHThQdMCu6nmRuCrdYquiGICCXpS1fRvFKC6miMbOUkkMoqmV9iI3yASnyPd08SMqnmlx6/hRArg9aJFdTQBRswI+2sIhGco6FTg3SKh8VF9OJQfqxr0CU3HFQG+56KbGoEMiJiKslkEqCyNUI0CO9FV33eQvo5t35sLcV+ORudn7lT7nde1t3iPO15/ky6JKAcDGyiDDClvmVE1G0IC1yRCQpAIazta3e7Tnw5/+39DedUvopTlZeBfo2BQDZQ6yaNUiBlefBulVnahHJRJ9dV8AJXFyKMW3QrvOR38LenUiAgNdiQh0wvX1+DAw3Byl0dm1eMbL8I2q8AHfyJGJtlDz4mdJJOCuesrRDaUwxRFLczNemcatZBPojI3kSa4q1DKQjICq9cUImNsPyDAXoJXorArAocOaHj9Ojs/8iS7s6/zhakif7Aa8PtrkoYrOYWqXYmE4zOOgqB9EvhDFIRtLyr9uXBvju97Jybd6vgI6qZ2qtM7osOtdqYdloMo3SZIhJ/NoCZQzb07fCujvj+TzrBv90limSgrN19k6sYi1WUVrY9EwprDhjIO4jG1MZcDoK6aSAUCCjWLuC340kf/MpNz3yfiLFOs3j+dyN6Uxozl05Jqs7AR1JEkoyaGrlNtKsPO1E1O7X4QzyjxSqWCALKpKLWmwy99+afY277Ov5xdY3alyOJygS996CB7huCpFxaoA9WSOre4adCbUBXxpZpFJhEloUvarseJiSgND+xag9Wqe8vJ6M2nSeqSRrlOVBdEdMHC1FWyJvz49HVadZd83CSdzfKpB/L8rb/+UX71V79HkIwi/RKN2VWur5WxAhjtjbG60SBqSCZG47TrNq52/5AzgIsvTmKZSUZ3WUjNo90wuPTKCivlNkYQsO9Imtqch51pU15eY2w8w6qlM7B3J3PfvYpfERx+vIfzZ4vUL8i3rh+6T5E+2OseaDB4KMnxj+1DaD5tx6NYazBxeIxXvrGGXRH0DrqkMpKY7KNRrlDxWshIlOX1JhFd48GfHuX6mQarTo2ppyfZ88AoZkIgNixGtkdx3PR7c9JvIZ19xJcBtbdwMt5P8XyLkj/JD6d/hb/78/+G5Irg2WdXuHI2TjyhM7LPZLXqc2C3yfRTDqYu8QqSmZtt6oFkPLODfQPHWV9ZZ37tPHsZYA97uR7M8me1V9/5BO5RXMJeNSjDPxUDM6EMFRF0DfaOYe/acPMiLM9BRMLh0R56kyZr1hq7TigYVRCEnwmdFOhmhwKUOnMJuS98yZ/+8YsMJqLkpOTgcB/7njjFv//GU9w45zNxHNJ5iMVCE8SDchHsFkSS0NN7u9NxZ7H35nqOO7Mg9yUd3gdNqoES4E9LeMMNUw1N4AIqqlmAWhx6/98Q2wXWPwfOAp9DFUsc4BalTDCJoAdJBaij4SNdm488DkcPGHzzTwMqm+JAQgtx8RYYCeVDSC2svV9rw0BLdfTTNUUlhlDepCm7EVAd9ZoMoLoIje/Art8A/aq6wTFUCqGzt5VRkcQtGijP0C1BDIBACPqTUXb1xNjdn0ar1pkuNHi1bpNem2Ff1mAi00d/3qS4d4D2XIOvzjdJA3Mrq4zPvET8wGHmfvhDGourvPHiedrf/Aa2EWG+Xma4J8OnH9zP+YvXWFiusojaho9FBKe2J3hjrcVQXTIdzoU48JoHN69W+Kkhi1/4/AjPX2/x3LkCMalsDhvlYMRQW/UicHUDIk2fvb5ibvNH0tyse0RX2wwZMCJRHeKTUWI9uS2NnQEY3jLr698iYkRJRfajiSh27HU8JGtlaNd0KgVoWhJfSiIRiRcopwKhMhw1VO+hUguoD3Nk2y+Rjm3D1Ey1gQc6dmmJ1Y3vE8m/RjTu4GoWjriJHh3CiBtoUZNOgzodTdX++D7Sl2E2w8bDRUOgcsgGCAMXj7Zjo0U1otkkCA3PkxjSJwh8TFPRe2P6CA2MiIHrqJ4e99ok8r4cjc0OgrzjtTv/3nncZifkNodFV7BHEVEvynXACTMbmwzDILSefVutKRflAJxHrc2fRpVv/QpKZfSGF7eMMnSHw6/qFBkVULZnBxo5iHIq4igY0qt0sx1LwDW2TOrwjuIByQTs7YeWBS+vwZ4J6N8NVp9GZjzCTz4YoSFjuAkNQ6sQ0VLkUlHMBLChxjcTXtsy3SifRC3AjfCayx4MFaC9pjYqzw1T5pcgNwLpBqy/AF4BGqcDjAGXVvwCxg6T5GAPf3XF4slFl793zuH7bZ9TKExlDYXtfQD4GVRUZC+QT0JZg4WtBUi3Jgbdxihx1E1u8WYvuEOJ1XEsOtDuOx2JTqbDpJsr7YjGLShV7+AQ8Xgv7do6AXFFk4rCVrpSRSITCWhY6rmhq8Y9rZrqlZGOqecIybnXf0ijvU4i7WFvYfN96lKLbZpyAGs+tLUYP/nTX8BorPLNp569dZzru5yZegUnzERIoNGy2ZbWmX7p+3x/uUrBV+/0phJ8/AN7OfviqywG3WGI6TCSi7FnKMZoRPK7r1ukIxqVhkXTk5ydaiE0pVgtH/qTEeKpCFE82o7DoT1p5kstDF8nqguuLdk4gUCPGrSaPr3CJu42mPnR95B+nWs31khFIZABjbKNHUg010MIsKVkfb1E3YcnHjtx/wMH7Hp0P8tli+efvsLowRSrZxusnHFJP9mHIVskenpJPVim4fv0be8jmTWZu7SA0KIUZBQCl9nLLRrvoZMBgA+yASIPeg9cfXaGJz6zEx+bteUNYoOCod0JGrZg7UILU8wzsns/zXaZZq2EZWuIRD9G3wC9sSLxbTWcjQ2OffkDRG2ba+cW2HH0ENX5NXab733+1hAqaHTnEvsvKWYkgeso51gS8KPCn5J7YYK232Ro1zYGR/exOO8z0KchIiB7JQOHdNLjAsMU6FYPGfIc3/5xThzbxZWrZ8nIJg9zAh/Bq9zAeo9DUp290gNaAcRckE0FU9IMBYGSgOvB9VdgI2QtcYGZ5TJH9u1kz/Y8Ff06mqZGXoRQK7HJUGijAlNjKJXpohAAVctjwPKIA+cuzfDlDz/MX/7Zz/GV3/sTGhswMAoEYDtQX4XyFajXVYG6eVjR9hodFAd3h1VJ2XWAtiRx2d3Qa+FjRcCAAWkPbv4QlfPvR20KEir/CMTPAA+idvzzKCtgV3jsMpBG0x7GD/53oExAwCMn4W/8Aly74TLYD3Mrm64lvPWaBqmszsaqr2pPJRDtB3NENcbyM7DWhKgPmRQkwrB2CKWVFtBcBf8XoeesMoZaqL0ognKmIuENCrOmW+1ZUwF2RnUcx2d/X4JdQ3lcEVBotDh3eYmaFVBAbXVfB/IVjz2VVaSA/hWDuNQIUMZ9yffZ8/o5krOTVOY38CUsuG1uNGdJZqN8+YltjJDg9557gxnLoo1CilhAjwb7x2N8/FiE5bNl8itwNKYxEMBKI2AhgAtrFtteWuJLD/bTlHleuFJl1QrYE97yUjg0ZWBRQtCW2CZ85rjBTz++i1qtl9966gLPLRXYk4uwezTGyFCSmLm1AqEocKO6wFJjgQEBLZGiz4hiaxV08RC6m2N9Kc7N0hqtoI6pNcnkFsn3+SSSarr29qm1Pb0KVhBhr3GKaGQQqRkEfthDAw0hBpHtL1KZG6cZ/WNMs03gzhPxT6FrAt1QzHh4OoGvEfgB0g0IfBk26FOaV6ATINHQ8YSPh4uv+cSSGcxYHN/xCCwfhwAhpEq5hMQDImpiCgMjZuA7Hrp2b7Vo9+xodAK+d0Kn3mnDEG/x/63v9VXF/fN1NUm2o0jldPf2et3OZ1vAs6imfG1UFP00cA5VjLwT+DfAGygHwUVlCxIoO7GEsgs91BodRk30ePjcRNV7rIa/66LUTYn3uI+GMFQln6Os1FIbnl1QGDzPh4mjMDAC0gioxC3MqE7MTNJGYGDRDFpY8QrRPMhlFe2shuc/QDf12ZE14IfAdQmn2mqcEqhxdVFFSWvL0N8Pe0dhp4Thb4K/T4MPNbH8GlpbkqgJdicFf3tvgr/2Rp0aqmYjh1I0CRTs7EmUTpxqqfGL/Je0LDqTxUB5ip2B6FBmQbdSvRMu7PACb66/6KTpOnUY7U2fF6jBTqsNO4jB3KUZfvfX/wWRXpfVlRmCiGoIlE2oppKWDam42nTMKDRCOsr+PsXU4aH4tKWEi8szJC0FH+w5eP9D0JYwkDehEeDrKVKj4+w8MM63fvuFMAXaFfsOuNP+vghJ6fPKTOU2IpPhvgynX73MN0+v3jaMGU0Q821S+QHWl4p4wGKla2FHo+BoGruG85SLdXb0xBnbuY0fvHiNXCKKa1s4VZf9o1EuL1S5sCz58IM9nNBNUkmT6fkyn/6Zw8x+/2W+9nSDsg9OMgKBJJZN8NDR/fzox68R0QVLbZ9VNyCe1rlw6dr9Dxzg+1EufvsKpXmP/E5J2/LofyBBqifKySePsH7zKpVzDXY8Osau49tYnbqGFXhkkyYDAxqLV3w26u37x/Tci4iwhuOcw8d/aTem5nDu2UVSPQl27Jlg9MsSK5bBrWzQ8H1oFbl2doGyEzBy0KTlGhQX1shmI4wdGmAgIhgf6yWbHEBETFo4eJpPdWbyPTvlDllcUhPU34v21vchUQEDccFCS4LQ2HXkUW5ceAHPtRGA5Tf49rmvcGrwJ/jcx/8c43sMzjzn4tYhlxOsVwLsgo+Z0ahFYPvAfvZmf4bDez/DnpNZEqkdfPpbTzK63Mt1bQk9mWc4eDdVzbdLJ/MuUeqnHaj90tSUusqgnAVTwtzVrpMBqh7DdiWvX5vmxJG9pNhH0b1GOswy+C7USt3jA5TRGUN9dxOl0z8YvuYCOB5f/w9/zF/5B3+D505vY3pmgYFtUK1AdRrMDcgF8OCuCM4Y1GMOuvFmJyPohM+Favyp6/ccHL37OO2FoCDxl1EbzrQAuw3uWSj9I/BPh6P5MJhfBu/hUJf/Dyhg9EWU1bANFa57BGW6pvGDJMoU3sPY8Bx/5y+0mT0Dr5yD+Vm67FpmiLYInaaNdVVpLD3Q4+C7PqxfBXEE7IJK/4z0Ko8xiCiGEAtkW40z7lVYO6eyGO3wdDy6xTTRcGJ0AmFblH7ggbRJsuyzUbF4obJC0Q0o0U2wbLYl2ihDPi7Br3qkQ8heLFD21OWaTaVu05Yq4DuW0Hjw0AgT2/tZXF7iexfnWGwHDAjIpEzON1wC4NFtafb3j/C98ze5vAyDGuyaiHM8JvAXWpwuBlx14ffnPE7YKzy6I0VuPM0PlutMNQIyKITFUng3s7rgcF8PiYEhSFd5/g8vsj2f5KMn+6FP8MCpx0hYq5jNIpEtrlmBsg3rvrK58jQouA0kgn1T20hpe6hXV7H9EpIMlp+huuEy11rm6A516wWKWXLNgwxZkkkHPVIgkFl8mUKXUYIAAt9HeAGmexiqBkW+Q8boQfgWegBCl6HZEuC7Er/tId0ACAjw8G8BJIOwBkMjwCcQEInH0CMmnhcgnQDf9xVzFmAIXVFQayFDlQQjFsWIalyrXLmncbpv6NTmvx15u1v0Vk5J5/8m8E+B3wj/70U5DEdQ6+kAKmLeiejMopyAJMqg7qy5P0MxITwC/F1UOvDXUQwJYSb1lrK8Gf52QFepbkepmET4e2sohyOOWsc2KtbxXklicAK/bWGHjkbMhNEeGB+E01eV4s0MRRCahmcL7KCF7yyiizyRSA+ankZGdWRfFS8ike2QZYpumr3jUHXER7FoLYbXejC87nx4fdVAFRrt+jQkJqGWAy3QiFQyqjFWsoG3C9qveiRmmhxFdQKvoZxDgWLxygAfR8WETKniQ4Pv4di9o3h06bc2Owg+asZvdh46UKiwePJNqTebLqXt5s/EUBeagSBMdVllhzPfvYQ2BCKlInqRHkWB14m01dpgpJVSRoO+IZXtX11UtLZaQkUGWzWoFcGZVhvx/crxQzuZW1hEuAGWX6U5e4lf+WeXkO+ATRiNCb78kMmrM2AVuiA8AQxoDn/8/DRr7e4JBUDblSzWAz47doSP/I1PsPg//AOuzlYQQhlG600Yn0iSyBkMJeLUKpJLl6YZy5scGB+gsF4hn/VYqrp84iMHeOLBFLYsMXelyNlzK+wdjfOt33iO5Xmbpq+GPzfQS6PeYHW9yo9fOEfU1Ck31XboeRKv6pHMbg28vFpepjDZILAlpRsVUjlIRm0O7s8wmNWYKraQTcFgf4Kr3z+PnnewkjHWZ2ocObmHkS9JojGX01+dwbMUm9R7JjpggTEoMLIWq5UG1loLLRZDMyPkhgdpCkFd2jRmCjz1796g54hqdiOXi8xeWiQ10It+tI+kZzE8foBa1UZmUsioSa1QwPU9iuW3L6i+VzHC9SKB6n9BJ8PUDLxAMmr6fOFgnt+61KBkOVw7+wydBd45m4ZfZtW6hG/rlAsBTluy3vDRhE5h1aOnLulpQOqgzgOPjPF3kv8AP60RSEk60c/xxIepR9Y58oEP8Nc+8TBzi/PvyTVEBESiCqpveaEeCLOzIlBtHwKhOhs3C1DY9LN9w7BjL9w8rezU85cmObp7lEh0lJa+RKoX5m4oZqROOZtEmdgdHrp11N46wK0+u0igVm7wra/8Nn/j577A3/u3/5Erz1sk27DHVzpfxKE05KL3SYwwqxsEd8CjpIr+B55ivYrmQztii1PEt0EkBdFRlRWwax5cfRHq/z1a+qKCkLRtkE+DPw2p/wnET4We1hyKF7ETsuxcMSjzsQaMoGvX+MXPtqnOwo9/BC8uQqWzTDZDPDqR0Y7xE4QOiDsD/FtI/AOIPgxRC1pVRSEZzUIyFhaRSrTUJEz/jwSzlspUhEyct6RDUNLBPL0L1qkmcLOoQkrzfkCV2x2LzT8rUHZGGngkDkM7E+Qm9nD14jLPzhVYoLtl5pMaPRmDQ0f6uTldZ/LCCoM5g88Op4k2qwgBX+zV+XrD5RJwerbGmZmLzIdU+fuAS683eU3APhMO7o4xW7d5dVGyvArH6g3yhs4vnkzw3UXB4oZFj+NiBLC3J8pw3yAHd0zwZxemma2XyNUku1drfHShxmPpOD2TK8R7R+kfP07WvH/IRS9dwINJt0ZWfZPk9cqz7ItUqPsCFx8NAxOdKpLpNvTUYTALG0WotdSX7RssMTLxfWT8WSw/huF9mDgfBT+K57i4joNrO0T8HfTKX0JKUxV7aBrSD1Q3cEcQOBDYDr4EVeUWEODjh4aRFvL+e/ggJKYZQQgD3/URwkcYEhFIdEOAprqKo+kE0iMQULaWebH8LNcbN+82NG+Se3Y0Nq+b+5HbYFJ0+6Z1XvsuysnowByrwB+hDPxtKMhSpz6qgoo/DKBqKDr25FWUYft14P+PiqjvQUViPFQxdCe+2kkJz6ECAsXw9RrKgdlHl8n0Ot0G0BOo1j3vVoTQ2H/sEfzUIFOnvwOoqPa2QUj3wGJB4Qb3jkFhRSo4Zt4kmUyxVi7TWilSWChS2tDxMeiPS6wdUFiGRlN1i5UoPdSJhN1pbnWcrRnURjKBcuy2o7C17Q2oXFaRs/1HXVJX1hBJE70dYMcDFhZhqhZwEngOlXBuoBLQoDJOD4Xj2xP+/qX3YOzuWSS3elvcqs3YPAguXYei42ToKG2h0U1Bd47pZEI64X1ByBUcprlN1AQLveGgrd63ve4mK3zoy0O5BvWmei8aU/DhqAZSQK0JybgqrqyUIZuH/hPgvX7/Q7A9a/LGVZdIIFWiRr4zV3hch0/sMrh2vsnTq7e/F9NAFqoU22/2ehIJk4/t68FtLjB342Uee2gPVW+R/oTD9FKFYsNncbZO2tXQZIOECEi5kgf3w9raAq1CQN3U2L23nytXZ1lfaZOPCsoFh7qAnj1piqsuiyUYj8CUA0cODjA/b6BJm5Hh7ayuFwh0GwJBu9nCRGDqW8NhXHphjl0f68e2PHoHHaxyi/ayz5WXZtHjAVotCk6Tm9M3aGoG9orE73GIGxFGJobx41Fe/k+ncRtdKMV7JqH9U59zeOqbM8QyEkOYfPiTx3FtBxGJ8dLzrzEyPsBKsUm9EdCX6mH3foOV622Csktyn4veLLO2VmVwKMCVOq3VCvXFeaYuFmm0LNrvkVPgvce+habHiERMrPbb40Ri8hAxI0pJnuV3rjbwHDfcw6IgJIG83fu7VjrHP/r6z/P5yV/mseM/B/UUjYoylEtuwCGpY+Ql2wZMGs00K4uS177dJimamI1Bdjx2kgf+m8OMHk3Sd3rgPbnWQAMtBo4dZgA6D1Qdox1AoaFgNn5oaAogm4F9RyA/BFYJrl9U8M3LU0vs3T5I9ZKBf8RjJbQR9tCNUtc1+BhwMejE8LsNYc3w53uA6zeWyZ1+lb/ypc/yH377m/T5AZcBacDILhBDEhFGa/1Q90pCRwN1Pc0azF9TrFeje6BnWOnBrYjeUhkcF6EM42EdMidh7v9OsPotsF4HuQa4EMxB4z8jsjZEP4e0L6BCkhYqdJam26rXQsX895BMXGKsBy6+DJeXoOyE5xsa/bfFcDZH+GSH1loCNUU9aCxDrQfqJiRakAirgs0IGC0C/gDWXusaOXfqkYBuYQ10U19bkCaqIqUTa3u78IxAoRd2An02jCZ0Prl3B7/+xjLbBFyRakz6tgkOZOOkbraYeXaJlAtLAUyv+5gFhyBQUOt/O2eho2J2L7hduHoaZZ/sAS5J+L4DZ65ZzAYqCJwRcK4JWsznl0WCf/DJMTTdYuPcJLG6Rznq8OPiEstXq5SWGgQxgx1oNETAXAsemBjkwMlHkHtPEotA5Ru/dt/jNoRyxjvgCIGyUTtr5SobtJxnMEiSZA9C5R9p4NAA5iqq5upyVZHBHO2DHRM++V5IpVoIWjQb38ZpJdDcD+DYFq7j4jkOmg8BMXADhPCRpiBwPfA0pOVBWyJV50BcHAIc9NDydvDRiBII1WTQjJpoRhQCoSiyTR3XcZHSD7MeAjMaQTNNbL/BZPk8z1Z+hCc9Ttyy+t5e7svReKvX78xUvNWxdzr6BeDX6DoZoCIrTZTRb6Ju4AbKa7yKcjA+CDwaft8yauIPoxbIq6iC7uOo7Mfx8NFGGcMtumtURzkWNdQEqaFSbqN0GRA8lOORRTkw71akDFhbnWW430V6arOLaqoh05V5qNSgNwHVElx9ySXuuQx8wGLAF7x8TuIXwXAh2x/BKAekfMF+oWGnfco9MOPB1SLctMNiwbc7F5RNfQF1L34ZWC5A/++CsQGtfthwIGPZmGs2bKjxKxbU4upBwaZeQDmELeAE6t7Fw/9NlGIo3fnj77dI7t4eoqOY71TcnZut1mY3AxKh2zuj8xkDNel8EC0gDrJIdzE0gZSisFsrKhz13oMxvLrEq9nYNrc66bYc1SsjCBQbxWoRTF/tRboOkQx4Q/d/+d975Tpu+BsHUzFGj4zzwuvT1K233kY8Cd+b89A8QfMOQzOvQ9H2b+2fIWQTDRjVJH5lA2f/AcZ3HqTpRXkivZ/5mXms+ZcAn0DCfLHB//jnRogWF/nGM/BbL8OG5+EDYzkNw3Z4cbKO7QbEpeSDB8fYl/C4vFzhymKblg8nh6JML9l85/sXieuSPSNx/vyf+ySryxvUCgWW5pYoXp/GMmK0IlvDE3gSTj25h0KjQeDYiJE6U+urtPQYLSfCzoO9ZHt0KroP6x6NSYveD8bY+/AITatMq+HTKLcUocW7ZIR5S3Ggdsah1gOZiEFl+hrLZpRWrU3NH6In8yDbH1ijWnmBRG8LnAjbBpIsXW/S35+hV/e42RAU2h5DqRhZucrN9TKpgRhrrzfQ8+8d/Oe9FBk4oCW7VbdvIQPGTo5nPsMfb1zHd6vo6GznEUr005JTBCyidgR1gwIka+4kv3/u73N68Tw/+/jf4mDvfoKmpBVorK4HZAOdxF6dsVWDWtHlxgWfZqFKJr6dYx/cx+jRBBK4dH2avWyRxmaTeIEKSsgOWcWdYoF3Rx19dhC274J4Thn4A3tU5qIVZrxvzK2R0KCmKWSRhtoDF1B7QU3XeCVqcLPhEEe9v4QK7nWCdB5qP/7Gc6/zicck+7YPcfbmMpoJE4dVbaG+ybIQQt0qEToZvg8byzB7WQW2ANZmINGjiDK2IkFdOVu6rgqB/IiAVh7SfwlavwjeOrSfAn4MnAc5i6w8C3ovmtmH1C8jrRHUTlZFmYttdEYw6MFmnkTCpbAKl2ZgIWwUZegCzw0tG03eHsDqFMIEEgKIaBJMcCLranPVUpCWkDYgG1WFeo4ErkPr19WA9aOMkQ26sN7NEkGlD8LA11YlCsQEVN4hMBCg6v4yEZMHto0xuO0QpcReVpNv4PZV+GB/jt2+xdJcnfRCkyGpAGlnUXaAJUGT8tYWm48axJNRtmUSMFvAQqExXJQRX0bFCaPA5UDZHAEwKWFnBP7yHti/UUQ/U6X30T6GH99DzhNIv8XwK8ucXaqx8xMHaMg+dtt1bp65RqllEfUk2tocRsxh8dkzXD19hqP3OWZhEuIWgqSOMg0E3fZeTaBCk+1UyWKiEadFW9muLZCLsGDBgFCECpoBsRQkU2oNGabDhvsdPOcAnhd25BYa6GGrFS/AlyA8H98J0EVEBTsdF6TEx8PBJgidjQCXIKzQ8KX6TzcU7abnemimRNcEwhD4noOu6USiCaTpsFid4lrjCm+0rhMg2cUYj4rH72ms7tnR2OxAyLs8h7s7GHfO283ZjElU1qCXrh3noArSfgtlrP7PdLtD3kAVdv8B8FdRCvBTwJdQE3Ip/I0Vuh2zJfAh1EToGEdZup7oKKpouYVay8soh6aDSxR06ze2GGx5k9Q2CtjNjVvP83EFr9DsUAm34bnz8Lc+BP1pcNsQ2y75yCH4zg9gvQIzZYt8PsrHv/RFRpsmpW9/j/PPVhgUMDgMD6bgwgJM1tR1vZPJ1WlH4Qaq83oc1TlW2FC2IWOrTrM3m8pp6xA2HUM5FlOoLJGH8vR/jHLMOuP2XxQ6db/ioy7K43YcoIZS4AHdlNjm+g8DpA6yov5PJhXrUrOhUhkiAtWyYpdamrcQNhgx6ImDH1VMVM0WDKQhnoKNVVheVK06dJSjEWjQ2EKAVITAeA+oazoHh/L07evjTy6uUXsL+2yov5/Hj45RKy2zdHYNUBzjGQF7TLXPaS7EoyZZ3ScjYaEVEM9nsXSP3Tt38vXvvYFIjfCLv/RF1hYWMJD88JnnAGjaPv/i9xbRW5IKgkjUxPccJLBQ8bDnGozuH6OxVKRSbnJtrYYf+Gw0W9SdMHtZchhIR2m1XWq2ZGm5xTf+8+/w+EefZH1qmoju4MVjpLNZTh4evv+BA3YfTLFRb1Kv+2QTEWp1l+SxDAMDe/D8Nj0TKfrGoqyWW9RW13FdiY9O1arjzxTZWG/gxH2yJwyqr3nvn7PhA0VoJjxOf2uF/NE8fUfH2dH/II4fx8yl2XV4Gl93SSWT9B0Y4icOj1JqSJavXGco45P0K7zx42UOPNzDakXD0+okh02Cu/Q6+a9BpAywNunOHh2q/puHeNl9gb7KEDJMQ+4Wh/lI7P/BBavFipQ0aOKyTItLOLyOZA2waeFwdf33+F+/fY4vHvoKHz1yBD0muLzkkbsJYw8ZZMcM8nkfzwZHDFBJ1yFuIDRBfdnlmbN/zBd55D242LdxMu4iyQxsPwi9nRqMQHUKH9oON66pYM8gUAugvdhFgkbowm49L2A1cNFRzkUTdVwDtSeUAEcDdI0NN+B3nj/HQ9t7iSYEO45KBsaV3rp1CbJbkyGlIsSYvwaFxbC3QFjQ7tmqL6Nmbm2o/DbQEgSe7EYSpQRLgB0FZxASPwGZj6tIQmUDvCvgtwn8FLh9qBBmGRVuHEZjmOH0A7SaZ7GDJqYuePY1yfk1cDHQdUkkI/HLUnVLvvM+uRJiSTBdUtJmOAKiV7KSDKhX+iGfByMHCbNbLd8SKmplaYoWsjkOvQdg6DtwOQX2BERzkL4EzaK6cSm6FOxbkJiAAaFRCgJ0IC7Uay4KnidlN+Y2APRHIzzxgY9zZPd+tm3LkUzX+dSxNBlnN07D5rdfKjFtKRRDJ1N2J4QbFGR7hwmpjMn5Qh03PDYbXk4dFQBOhLczSRfm7QGfiMEHTdDSw+z93KcQExNUPZ3S5CWks8aRR1u4L67TMsrMrde4OVvGCQIMU2dqcp716XnWBdRsuSUV3aZrZ3bEDM99I3y9wxPgUQmnR56kUOwN9QDaVojolnC9oJyOzyVgsFfZB1oA9WSJWu0VfO8kgQwQmsD3AvADAiEQmmKXQmoq8+lJvLAzrIODi4OHBbh42Ldscg8XgYYMBG5IpiIQSE0j0MA1A3zRolKeZs2+ziv2InXp3vp8C5u6vDdGzHed0Xin4zY7IXdmP6qoGzWM8lQL4Qm1UE6DhoL4HEIZrqsoJ2AUlbq7Afyj8LOPo4zaOspxcVCTMoNSlp2Adef3N0Py46gAdSeL0URFeNp0m0rfh75/WxG6ycTDn2Jt+Q2oK1BtT4/qmfG5D8DlizBbgKUl+J++CT97HD4VhUYAo4fh6AE49xK01qFU0xgeEkQoov/kGKX5Km5VktkBR04lOdEyeP2lFq++4TIfqMn/VsjrWnjNWdTC7gfSy6D3q82uWIS1lrpnnchWB9XYyQqtETJPhGM1gepzEqHbc+O/SgkhT7f+35y5kNzOHNTBorXpMn+kgAocfGwcrCqvPVuEJvhJVATLg/UiJCKqwN+yoVEFGYdYBBJRcJtqLmrh72lRpYSQKtt1v7KtB2oV5STO1Zr86jdfua2liIdizeuJCEqOJGVq7OnVKa+VSZg+x/sFjaokIdW+fKENngYfPXGAz39ogj/6rWeYr1g0gavlNjcCnz/9n7+KGUnw9T/+M/qzKfoSE6QS3ZMPJFSbEhc4+cBxPvTRh/iVf/UbiEASABEvYHdPnmKgsVSY4dpd+oestyQpw8H2VZfYmg0XrlfxxFlkqchKzaXleBiVOvMb5fsfOGBpxcV15xGugzk2zkaxhZ3oIRuVmI7E8XVqqxvU2hZrM3VICOyGy7WZdUwrINOrYcQ1+nfEsG9YWMX3oSrcFMqQkarnxspln7VykdySxeijffRs207bsWi2fNJpg8s/mOfIz04wYAqeO/calY0mo/1Jkn3DnPhslI2laXY/so+F6Xn0foNM+t7YRP6vljc7GQpB3maDM/5/IEBljVPmEI+PP8i2lXWu1haYQifQRrHF41T8Gg1mcHgDnyskMWm4kpfeeI1UcZw9O1IkNI2XvubwQFPDbgvWlwKSCY1KRRDJZllecNhxI8KV6/P8+Np3UCGy+5OhfEiO4qjetLfBiDrpw7fYiIRQ0KNYXP0fdLAcEgZ2weKMgmDVULq9E9jzUPtuHaXSAgm2L2/V+GVQ++MGKjDoAnsleH7QbTvk+aSSgr4xiaZxO2VteO4BUF6D6Quq/gyUgzG8E9ZmIWmq2jR9q/6tjQqXy/D/AgoXXQ1T1NukivzUA0XtSBZVWTiHAnDPoaoJP4xqAfw0ko+y0VrACi4BgqUVjZVVH08KIEDXA1xXo/9QnOqShV2WvCmyGrQRBkQcFUzUhUYgM+DGYLUJSQ/cvKoKNmJqXTceh9SPIXpFwasCCd4R6H0CxD5IrkDir8JiURkoCdQ+tMVIqASEGbAzgJKnkoUnDMjmdKyhAV6/UWC66WGg7KohXUeXEUStzvrVN8ib80QuLfDtBYdrDZ+p4HY7QwdiQmDdUR9oALWGx2Sjggf0RnUe/tBDLJ+5Ro425YaDa0vmUHNxN+rcPh5VEPFHT+0l2Vsgn6zjFF5Bjkpyw4exXlvj5d94mqGMxIvA1GyT8RNPcvKnD1ByI8yur+BfO81rNyZZ8VwO5Ay+WfH4p/c5bg7d6dZx2jtmt0u31skASlQxsHFp4EoF4+wE1g3UVF2wIGcnKcwPMbFjmnQCAhNipqSqXUFyDCEEnuviWy6BFGimqW671DH0CMLXsOwGXpi58HCRYVZD2bwSHQNflYETNRPKWZESKQNFSOBJAtkE8zob1g2mGxtM+tabACIN2rTe1Cr67nJfGQ3ozuXNGcLNx4g7nmt3HKNtes9FDbSKJXUh8J3PeCj8fy38+2HUzfkiSikeRLEMZMLnq8DfCS9qDtXEL4VyVjqTQqB0kI9ySNJ02eE6bHFa+H0i/K46Cob1XmQ0jh97kON7TvH1G6pqwdRhfADmV1Sk6Mu/AKszihKw6cDCAjx9AT4CRA2dPckAf5tkUQM/6lApnyYWsTAMi33HTPwNj/Ejkvw2Hy0aYffOLGk2ePF1xQTeSUd2xsMUSsk/uB0ycTjRD8nXoKcGyTrIFfC3q3ujX1UR7Q4BRpkuEUZu0/d2JuTXUBvax8Nx/a9aOhuzTncydKjKOpSCnUGTqFBLJ9vhAzasTK2ju231vAoyCSKjsh52TbEbtnwoN1T0zg1gR06xjLUcRU8ZWCHtramKPD26+Ov7kZyEwSjE+sGXUHMFgdBYXPepCsFwLokZuBSrNjt7ouRSGrXyOqs1g58+uY14zwbnLMhkk8wVW1Q9yVAiwj6jwTe+/QKvbFg4UnX8rjXbeOEmEtgt/vX/8k/45Kc/yvFje3nt7Bu3zkkHPnpwO+m9+/iZn/0LTJ15Di2QtzalYt3h1TNT9A8kiMVMGu27u8WNEPivCVXTUrcCyqUSgQfrLmSzaU7u76VcfrOjci+yb9cuWlaV5al57OIkru9R27BpLRXY/cA27MU2RmChI9j18AAbDRvbq2HpAcntgrHeDGZCQ6bSLJrLGNs04tmA1k3w7y0A9M5yl47uQRVKrzaob7xAetsFdh0ewA0spi4WWH7FJ90zzc7Pj5KNScptwY7dOaKmxC7OcvmlKZLbe4jne4nKNkaz8B6d6Psrd0YidQboEX8Vlxks+UN85gAY0vbg6hbL7hrrrJPC4HM7D3D0kVGevVTk6ev9LLYPAyWyaMywgS1HWFxsoq1Kto9EiPRF+dFXGti6hpk3iCZAOALf1ygXBS/8aYUfXfoeK53ih/uUvhykk4oCeq0J5SZY7ibsfwe3dBcxo4pprtlQkdBImBmQgBmHoR2wdF3pcRsVnPNQe1sVZehs9l+s8OGjVOAK4Oka8aiBaKki0xgq2NQ/0EehKPC8sir+FtxiYiJQFOorN2Dmmvof1DHDu5Th6NoKsuPOQ250S0OnLqouYVmqTa5qq0GIOdBag/IN8GeBNUztOpn4AWryIK7/h6iOWwEqdPlZVLXhDAYmKXI47MfHQbKGJwuoEFqBoSGXtVWbjZs2+BIRDTNQmyOajk9EU/tsFZDGCK3KcXBNBY0SWdCj4EeUASABLwG1fZDaC/0uFCyQh0FrQXoKzL8Nk5PKcEnSLQaPbm3oHsrqzNR8ZgN1P4UP0z5kNnyyosIjwwlG1lu8VPOwgWWp8bOnHqd85TIbhRWuXb9BuRxwnS6qZLOcGu7lMw8e5dd/+CLLltrMIiiHN6mrQFY6ZrCqg1dr4BmCG/koqbhHpu6T0TQu1wJe8uFUAPsdOLADijMLfPeswzbh8+HeS7S/dpmGo1EsBTSakssWmHmNo48+yujJj9EWNsdH9/LBiR1c/ZrGhZkpNnw4F8gtQbs7y7GzJOMo26hFlzWf8G+VgDYW+4gTR94KfLPpOxLEeEg8Tsoep1lZoSfTwowqSnyhbyBlC9+J4DbauJaLHjHUJihNBDq6ZuC3XQLfw8cPnQ0fmzYudvirOpIIHhIhNHTTVHUZSHRDx5MOtlzBib5GsbHAfNljMXgzCl1HZw8HSJK5p7G6r4zGZsfibtCpzf9vrsW4U+SmRxL4+yg98W3gDF2Mm4YqCGqhHIoqilmqU3droprqXQm/50so5+EpFATqEZRTcQY1sbN0Hf8O5eJNurXDJmqydI4J6EKKOiwc71auXD0D9jRZfwPPhO19MJGDSwaYC/D1a7Dow6FdsL0Xju2BlICz52Dqis/wDo29uwUH9wWUGwaGvodIxEAbqHHop210CRGvhqd5SKtBRqvzxEMGN2+4eHV1jT3hWLrA7hg89gQ8dEwFVwa2K4pd53XIWKp7uJeG9LD6wMI1WJFdzupG+Kjz5k3fAv4UeB5V5PUX34Pxe9+lw1bQR7cWsJP6itCdFB2gsq/S/UEOFm/Uu8XjBtAOC8Uj6jONKjgC4nFIxFQEz7HANxStbbOuKCeTPdAbV1msehW8LRinZ1bVqcZrCv6UikoSWsBQFI6PxRg/doDTL16hVrJZKNlULUgndNabDl99fpqMDus+xDQLT8BE3qTS9Pidc4tU3G5h+d7xIdrtNnPrFTV8QcAPnnmaF156kb7+HIWNKqmERqMVEDV19kY9biytcv6l53np+edvI/JypGSjaUHFwPWkwj/7Ek2obqWg9Kqm63iej2EKMhmDVNJk22gvfbk0g8tNVpo2e/cdYWVm/v4HDvjAE4d59ccvMLsSsKEHJIYFpTdcRN6ltTRH/8Ec4z0uQ9t7iGZ0gvkGRibCoiwwlo5j+A5TP3JJ7nLw4i7ZYwn6emBuvfWWjkZsUMOtBvga3Xx8Rwndi3RSsRFw6y3qDZsr5zdwCj5eI0AGcP7PJtk+ZvPo8QG09Q1KS4L11XMkIjqrVyz27zGImzZzZ5fYuNCAf7ul4fu/VAJW6Mk4HBv860wtTfB68x+iIdmV2U7CiCNdiJFVUNlKm760wS//3ASfnx/hd78/zzOzgiPx3fQFa1xpX2eeGTwvx9qC5APRER79UC/NFqyvS2q2JJWBekPSagp032S78QhfGPnbWzr3hg25jIJWGlFlJ6+3wXZVA7hbgY67iONAdQMSSUUyoaeUMS+00KjfA+UlCBpqL8ui9oE2ao+9M5bR0eU1un2r8ANiLYdxugkDHShV6mi+0l0yps5Rhkgiz4KZN2Bp/vZi6fwwRGLKAQHwHKiugrPVvjOBhAuB2qSDBpgrijO85YEog74G/kXgdRJ6jHykScMuoTASvaioUZxuT+oxPPqo+C4BTwKHkJxF7aAPAS9QWFti594YM9M21h21MrcCVlLp+IYmcPP78dwvIUtRNaq5fognIJuGVFIdn0BF7TShUlq5mHIgVtqQvAzG34Ubk2r/idGFS72LKOhrFf8W6U3nFl0C8EBfbZPT2vQKcSvZX3Rtbj7zu7TLG+TTUK4GLKGyXneT+WaTWavEEyNpbsxsUJOqLYGDKh7facBjOfjuisfcuSs8sjdNcijHlefqODrMojLrvSi77rQPG/OwI2hj+8q+0EoQQeLj36rrmLVhd3KM3MgTxLwI1773VYqZGNs++QRL8+dpSI8HJrK8st4kYdz/AHbs0E5c0tz0ehA+79QZNwCDgDUqxNBuBdw7ZXwxYD+9ZEgj3H7qpW0EO66rOs0Y6EaDwC3jtnK0623VcsVQECwhJboE6Ut810fHRHX/Fri4WGFlsrKpTQQaLj4JI4Zm6EgZEEiJ0Nr4WpGa9jS1ygYLDVgJuGvOYpRtHOGBkL3qnWVLPAVvVauxeaLemQG58zgflYptoOA2/y3Kifjfw+dRYAfKQM2gYgg3UQrvaRTcagBlD46Gz59GGdBXUE3jzqJqOjZQ67djXFvh/wm65EQeKuqeRGVGOoq2s8+XUQ7LuxUviNCITFBplxDABx9S2dxkFj7/52HDgh9fUbzKNmCloboEtRpU1mBuJuDM89DfCwN9Ntn5SwydfILhmImx+AZBKkZ9MIqb7iPImwR9NSI9dbKvLXHjdffWdXUKrlq2ciYK8yDzkGtCfAy8bYLVloFx2SNYkcg1qLagJNXYrYZjWOfuxE4d6QT/K+/B2L3v0oFFdVioIqiLa9NtOtIhqwkzGTIInYnNXnjHm4vS1UZZRUUpXTDCKFRvr0oBl8rKmHDqKriVzii4lN4GIyzavF/p5AIagXqo8Ik6wehUG268hh1016zVhPWmuoOOhGoYbrlQ9RUHvu3i3cXo3T8+wIWpuTe9btkunpsBWaPRUtZ1y/X5wwsr2P4SC/Nz1Fz/ViIprkE+prPS8lgpVAFIRA2k9Hnk+ENcnLxCrVFn544RRoaHefGVc2wfipNE4oko2WyabCrCU3PXqbR9/vjrz/Dwo3vuf+CAUq1JoWZhC4hJg75tKZaWK0Qygr7+HJlsDNexsWyXm6+vsV6wGJiI4wZQshzins7gA3mkJjn0ZJL1WpWYkKop6bgOS35Xseigm4JTP9PH2vUaq5ZD62aAEQOrCLJKF7/5dtJZfC6wAm7dx437yiLUAQe8mKTmaRzozXLg8QxGNs3ZVya5+vQGzXmf4o0iieEIpck29upb/9TbSZTuEnm/RUMjxyB1Srjhr0psZutfYSQ7waGJQ1y+FGMUmx16gqjn8XDfCD3rUywHUWY2HH7zty+ybaiPXfsGeezBbYyM9tOs6exv9/CIP85raze40ryOJVssTS9S8Pfyqce3cWw8ysqUz8WSjeNCcV7DHImwvfcAR3bs39L1rDSUg9GbUsHuhAZZDYqugsfdzek0CPuIBuA2oFJRtmssptA4HYnEYceDsPgSSEftu2nUHrrMWxsCmxMocVQ94/7wsxWU/r++pCA8+Y4yEcq5caow9xosFe/wjwRUC1Ba4bY3NO5+jfckz1+FdQ/8GeDr4MSAJ0AMgliG6FdBPwV+m6obUC330o2/n0ctnE+HJ6DApZJVXC6gyPOvoayACTpWfrutMz9nke8XFDxVZ3JLOoECQCJoagPAP4bmfvBqiso22QupjGr1rmtdA6lz3wKU35OXEPs+2P8fmFnoslB1MicCtfC2yDrVOe3OreiwKHVOIYpgm6Gx4vtYgOF5XLh2kSd+4oOc/v7LtIIuzHwzMWM0fL5Us/jBU28Q01UidkyAKQQ30TETKax2lWvLHsNA2w5YnqzxeKmFGMrgt1wihRbnUP5XBPgBcCOAgxKGNXhgfxInGmf5egmrHajsWDg8piXx6iXOXf8x18++QdQPeO5Hr3DdD+jPGUR1jwcmetgbyd33uHVqgDtjl0Kp2wClcpO82S6ax2cQ/9bnOvtfDFhgBVtcZMQbJlHeT7M1RTYTYBigR3ycdg3PiikODF2AJpVr5YIhDMUUFWa6A1QhuEULG5sYJh4+Rkhpq6MjhMB3HIQm0bIl9IEyljOJ296g4kHBV3pi89oVCHYzwXEewcAkuMcGLvc9Ne90Jt7KkYA7lMsdzwOUQ+Gg+l2cQ0Ghcqhoy3bUjRPAKygjP4cybi+iJnbHAYmh7MFFVFHyCCrVO0e3Z4aFcloadPmOBV1O6DxqYnRSxg1uV7IeynF5t+LbLW68cRYZeMRMFYn64cvw5FHYd1wp7h+cU4ZfKqngpOmMgnHWW9Cuqui3m9dxjBjVWhH33Pfp7x0ivuTgJ8tEgjRUq7RycQLfQQQGiZy8xdDq0nUOWgG0r8BGDawHIZWDxhL09khiOY/Bz+ZJrpmIRZfISovZusMFO8AK70GJkBbxPRib/2okRTd70dEEm/F8hK/XUbuvAXeGhEQnutfJbhgquuhY6j6mEqqZWNNVcKl2FaStmFciMeWQNG1o19kSR3pUh4gp6EnqWJZEtn1VfIYK9CGV0jj+wHGe+MwH+e7T36LqRCktr+CtVyFQQbVUzKDa9t4yknrh4jRjyQTLdDcZUEW7leoytnu7l1QIAqICrqxVbw1lNKLxuf0JLs60Wd50bOBLelIJTp9/FT8sTu7rz7C0ViSbjJFP93B1eo26VebSXBldwM7+GBEvINluUpm8fP8DB7z83ItMXSyx69EhevNJcskUM9dq2G6AcG1GhtPMXtQpSbV+rbpPJKaRsExKZYd8RPDAY9to1Zv4TsD0q+vYaYEfNUjtidGsN9ASkB6ME5g+Owdh/55Btu/dxfmLl5lcreG5XZYeY1xD1APce0mpbnZ2O1HVcKC1hGBpvclXv7JEsj/FxIEcc+fK1FYcRAKaNYfUSJxt+3vp/VDPlsYui9IH7+QXvRsRaEgC8tpePt3zF5kuv8qsPEMlKNCmjR2sca38q4yl/hk7jY+ie9/n2cq/Zs7/U6xGjUIwh0kPc+IvYcl+PrN9EI6ZjG3T2K1naKzBK39sk7KT/Pyx4yxu7OWpqUvM+Nf49s0SSwsFHkmNkvQk1xobFESK/aN9NExJxI2S2EpRFSrDOb+hyEAypsL0NxrgdYIVm6QTCY2EDwtU1rQE1RQkk2CEARApwxqOfuCk6gHn2WqP63x+8yrdbGh2xEAVkvsoByOJgr4sAQ3HV6UjEtrtsGdGHZZeh+XaXVSHVNmPW9diQDIH/UVop7YwcACxBuzMgJUC6xQ4A+DMgPZD8K9B+3FUJecECqxdRlkMFeBweCWrKOtgPBzRDjvIDbql9KN0ydx30Kyepllt3Gb8CB3kLfhUDPgMeMdgQ0K0BdkRcMuqKl86ivFDRiAJmiEIOnRGLQkrFpjfA/4xFFKgfQKG+oA6yGWIzoK20Q2QvQvpoFA2xUB4xNT4if27qNkWFycXVNNITbD92CGmb6xzZqNJJBydO0sZO/p9EFiXqj+hIcCKafRmM7h1naYjqYQMh4ciOo+M9nFgYgBprTCz1qBse9SAk6gsS5GQA0OD2bjAykDR8dhr+GT39KKX26yvNLjoqq25sLTA9T/8NQJXUb3bQNoLiAFay+PmrEdjbwrXvv/BM7idxLJjU3XGTtAlFDJQM+FuxfGd4yoEbPjXKayX+HL8SfIrMaKxFo4LVlvDtSMgJUZEQ2oCTdcJfIkmNaQU+K4LgY+LjUObNhZlqggC/JCmFiQBHjoCpIcIAkSqTmL0JnV7EZsy5RZUfDU97TvOcwfjHONBkqQIsNDeD0djU8Di1vPNcmeG426f7bzfUVyglvSz4QPUBHkUtZxvoFTDL6C6gU+i4DgzqGjMFF3D+TrKsehAejqZk87mV0QFmjs1GPHwHHpRSlNHqZoN7t6E8L0hj5HIwMPUFM7w3CQU66oQ8NKUgtE8eFzR/rXqsGcC9u2GTBSuvQHf/h2I5aDvw5KerINflLS8Jq/XpknGwIuDbrfxdIlW0nClpG0Lmm1VFrm5cKkDGSsm4DOnQGYVA9JLZ2H8Adh+UHJ6Gh7/yEfRq+tk5q+zPVXkX7zcxpNd5imf7qbXrylDtiLfn8bI76t0HIo83WwG3F61JcNjbNRE6UCsYuGjALgqQOU1UGEdDwWjiqrP2zVIx1STniCsqAzaCrKGr7JMdguaG2FEu3L/l/J/++IBTj24j33bTdYuXOVv/5vLrDdv3/LHUzk+eWg37vwMtZtFSnUH3+taAIGEpvP2sz7TauE1229aG5qAdvt2PsYI8OExnUIbzhe7nzi13eDQoMMPrtz+LZbnY9WaxDp0cQhefe06gZRkYxF++c/9Av/+t77Bq9cU/sKXMLVucWJ3ikFdsrbRZCty9rkCxx7qZ8/D+1k+N03/gSHG965TrVkEts8LP5jBRzLWn0SkIb19kJ5MjFxPkupalUhMZ+78NWpWQFCL4TV1hk6liY22WJtsIy0wd4JltPFLIHaaTC810EQRo90g3gvVa8oGAZDNsI/CvUqHqaHj/IY7YtCU3LxeJtEXYfduA626TN9ujSCiEYkIEqmAxVcKHP7CA7RXtlajkUP53x1d/97ozM2ic1D/FHW5RsY8RiZ1mHErYKd+mDYr3Gi+xnXvKoX6OWYW/xUnh/4OLyxt43L7B8xYS0yk/zxGfoRJ63V+/hMfZ2L/MKkhjebSFH/2az8iyxGEnaFZ0rBrBlUtQivweCAYYD9ZHF4C1+VKeZ04EKdBSjaZW7Twyr1kDuRo92yRZ9QFx1XOazM0xu/WrHNz3WOL2/HTQUMRKqVSqiVDxOw6GoEG2VEwErD2BtQ2VIS5zxQUPUlb3n3fEyjEAKh9cx0VnNuO2n8NoKwp8olYAkQF1i/CsvXWCQpND7PBEswYJFvQK6CSv78hu/V9HzmF4Utcbz/S+SiaF4DjEJRdmGvDUgXk06gwZQVlSSyjaEpOoayDKVRnqD4UxUka1WGiAnwGlfkoh1fesThC83oThEPEQTZQxXnsoNuyNgGeoQoz9QhsXIViDRYSoCcgliXQEkBMeWuGhIgPid0Q/RrEM4q/OJMIB7YJiZug/z60fhtaHY6jrUknLoE6Uz6fTzOie1Soc75UpRBe5t7RYXY+cIJf/ervMytvd0zNcOTmUbC7jhGdQemFoYigaAVE9Rrjuo7fVLnIxwZH+Pjnf5GHHt3Nuee+x7//o+uUWs6tIGZOKDSFR9g8ORLjUE8vZ4trrDdsXsdmANhnCGJCbcPhtsui7XCMLvTKA/oMGAzgpg/tcpuCM3vf49UhFOqUb1bp2r5e+LwzLfIafP4ArNYVG2hL3l5e0FnDI8QZkXl+d+a7/FyPTXpABZ0bNYHUBUYsgnQ8fCnxAh9N6phmyDqFJBA+bsg0VaPCOgXyZIgQQaAT4CPx0IVBoAV4sTpGzxtU20tUyhbVqs6apwrT7wRSDDPCwzxOkiwSGx8X9649BO4+VvckHU/3zgxG5++dTgh3eW3zezqKTapTa7tZ2qi2OZ2b1LHfSqjO048T9nNAUeFOoJyOpfA3esL3DJSqGEM5GzoKfdlPN5ZhcXvWseOJvgvI4z1J3ISJIfjeKwqv//oiPP8flX7RpKInHE9AfwB9JwVpA3aOSB79eZi6DkE0IJoOcBDUa5KaUBAeW0BESiIuCCdA2jB5EV453+0h0ikxiGiwIw1DwwqqNZ4BBuHwR8BPgdUL+b4qJXEJw29jxhp8eIdk+Jxg3pK3HAkdVQ/zNwx4ZAc0sjBbhwursFRTiub8+zye70o63I4BqglfDGQnoBVFXUBnYnSwIR3vqkrXy4qhJp7c9MiCFleORGcxyLaqvTAslZ3ymxBJqqJN31MMX0GNbvjmTvzvPUiytYRbNFms+jz1w5ssbXIyOkmYYrvOv/rdr+GFrE93E+8ujdtMXSNiaqTwGNBhsuXftl40ILijQWBWg31R8Bs+C3XojwjWHMWn7i67fK8gb8G17pRdqSgDWYPU0ABPn5+l5ULgefzav/wK09XGLQKmjly42WCiTyM7vDWrJZ1PUl13WDw/yY1Xqhx7KODQiUFW61XWp0s0mgaFdZt2PYIc2Mmu0Z0MJV3mb77O8ECGpheQHkwz+wczbKy0SW43MSM6pQse9oyP2a8xMKJTvS6x6pLVusmF5xaZGIzQO5wEv4H05a377q8Fb1ZId2Pj2CxNuo5zDIyYRs+pKNpoD0nZBmGzoevkUgGD+wULy5LygsvYoRSBJ1le20p5ZJexrkGXGGKraJi3El2LsE08yrxd4ZWl8zSDGYSsc3zoIX5+/ye4sPQs31n8T0T1l0mZT/L40C9zdXU3mmhyaNfnqdUDZClLWu9h/UaV3/76VznZ81Ea9V1AP0Yki5cxcHQNSzcQMdAxaG6UuFr/M3IZE00copcEETuOIX1qTpupxjzWG2VGk6OoaMV9it/9+3ZJzCG4LfO3GWWAr5r0rcdURiObUeQSsuNECEj1QOwxKJ2FxSWQm2qu7navoijVZqNU4QZqz+3Am6dRNRepLERWYfkSLDt3/y7dgL5RyPYoBirfV4/1NngZ1dR0KyJWVchcCxoEtAlqJtTSUExC2QB5BvghykpIgPYIiKPg2yiswgLKyljgFrsHfaguEC0UYGwXyvHI0m1qMYCyTGxAh0AnqHfwEqDu5AZqM5mBYD/EPgZjE3DDh8IU+GXQGuomSR8iduiJGcorJI+ylERYPe8rtpBoBngAgkOQ+xyM/vdbGruDOkz5sEeDxUBtZU9qsCuw+VrV5bgR4PpKV0sBDzxykLm1adbDwpTOfY6g5uE03S1LD0enD4Ua3bAlR5OQHdQornuUgJ8ayvHpT38KrV7lT/7ZP+HphRXmXBUYbQDJiMaXHxnnX700x6gnyaXjHBvNs7BYxm14DAG743CjDddNyMQ1RCVgLVBqcD8qYF1DZeJuAOMDgoQlaTbBdNrsz99/RqOTtOpApTrxxs6dr2w69lA/fOhhlbUUT0FttRuI7ZReRVCl2v1mP5POCj+84bNrt2pvMLssKZXPMxzdSzoyiBboBL4HUjkXOAHS83F9CxebNhYLzFGnwSD9SAQRImhoCBTLlE+DSP9VgsQ0juez0YYbTZ8qb4a/jrGdJ/kwWXoI8HFCK1K85QZ0u7yrjEZHNjsbd/vM3Z4HKChTjrsXWW8+/WXgN1ElWHuBn0JNmOvAS3QnUQzlte5FedQvohTjtvD4ThZDouzDEl3SoA4EqIhyQt5vnHE2A6stxSqycwSurkChqCAzHblpwMoqTK9LFpZgsQ59gzDWCwMNiOQgPaxh7owiHY1208NpuLgFH78Ka1Nw9jxMrUgaQbcdhBNeX06q2oDMXhCPQ0lTDHu5Q2oTstcgtsNHM1xi8TR+T5V0ts0DCcl8qEmiwCeAvwB8TINYWrFU7bbggz64CbXxvbgF+M9/EemkvcLCPUylTJF0KriUBHSbsfgoi6pBV8t0isXDieZVUBAsSzXdSybB8RW1bRCAXVdUll5dQSTifWoTtl0VlaSECuVsMS2UCBweODrKxRdO88JkGylCQ0OqLt9eAC3fe0c1oQnoz0XI5pPsGB1gZ38vyVgE26tRnbpC+YbFqQGTghBsBBpOooeHTu7i3OnXmS9Y1B2PuA5PHIwinYC1BZcP7IoRzyd4+lyJiQTsSsIzZYWL1oRgYjDLzdXKrXO7XLO5XrNJrMwo2k6g7gWcL9YUJbCuYUjwggBNE/iBZKkieOzxwbe4qreXxz49SjQeRbcCqnt8FldukvCb9EXBzQX4wx71hk4slaFv9360Wom4ARHdwDckVrtJXAvoPZGkMV3FbrksvlGmseQj0pAbT7FjX4zciQT1ags9msS9toZIBLR9DdFJX3RqM+50KHqFohh6p2BS5zNNiB1Ogi1pTpaxUha1pQC3FuPUkyMMp4osTdXoHQC9LmhV1skMbM3i2y7UYyVQ07dIN+p+P0mZt7uoefcadS7hY7LgdjyqOnPLZ6i4GjuH9rJ/fZTDKY3l6n9k75iL3f4Ak9Vplpevk0/0smvA5eqNr/HC7DM06hbZwuOY9JIa6CE3mMFHw7MlviuxHUmjHjCUzpPr/Uek8ylS+UFKiwFOQSIdQSxoETh1PKdF6eUFVNT7vZfNbDbdEblDLKgXoZxSGY1UUmU0/EDZsXYDqsuKZCLTC44B9vrdvohbTftWUOaugTLellAOZROgB8b2gbYCcxdh2Xuzk6Hpin53+z5I98DC5W7HcDfcS2IjqkZtK+J/ZxmfV4BfBSZV+ljsAB4EGdZjGB+B6ElIDUK6H9bXoPZdNO1VeoIB9NhB1u0NpAyNehZQUKnpcHAeQhWPX0AtzAdR1sV5VA1HH8oZyaGckSpqQ+g8d0AugSwgRoeRwTaI6bC2Dn5dbf6mBRGh1r5mgOFB1FHNLTIRtUlEdEgItfnq4ablPgz2v9/S2H06Bdk6fHYsye8uNbF8qATw9YbH+ECOS+Ua120Fj9vZm+HJYzv4/g+fpeHenq/soDQ3wyY79l4V1ZLABIYsmFvwqDsqCByttXnmT77BSrnKnJRMoeabhppvUUMjNj7AQ1eK7HQtNnpNXrm5StYJbhH3bHhqyxxywZASIWFUgyNJndfrPk2Ug54Iz3EYxTR12Yblgsvgxv2DPaOorb/D00B4vp24ZQfeqAP7B2Egp/b+Q+Mwtaaylh2ihY75UMNFBD6fz32W0/XneH1ymd4InC65rLqXibiT7NV2sD9+mLTMo4kEuJLA9iFw8GhjY7HBKrPMMMgIOjpa+DAwMXQDV9qQmsPSZrHbPgtrMFkNe9p4t+uYMcZ4nA+RFj0EkpDPqg24CoJ1D7JlVN+dWYrOXni31OtbvTeAcgzeDnrcsfnKqGWfBJ5BKcDX6FJI70TFI2xUPcci6ibuQtV+RFHIzE46S6ImgI1SIy26tLfx8HvfKybKOyWRgJ2H1G9FDaWAx8cFly9J1hbAaUHgqcXz0k316Ig5CxPjqvBuWFPQFjfjIA2BFokTHYpjpg0uPL3Bd38QUHK79SadAqnOJlCUCpIxdRWyORjqh9489PdAz0NgL0CyBHZCp0/Po69OUov6nMgL/rQkMVFO307UYqtnwRgQOP0a+g0fcyDM/uZhZPF9Gsx3Kx1t5qImySjdQZKoSZSmy0HXSQel6dJLQLeDeD9qomlAH5hZ5VAkU5A0oFZHdQ0vg1OAoKmee1I5HEGHU9LddA5bkIV1wb/79Rdw7SYn90Q4qaU48YkneOPlM/zRM0s0uT0reTc5OpbixIndPLI/x8raEnPFJs88N8uusR4ee3iIzIDG/HU4W/aJy4B0XGcg57MwOcVMoY3j+BgCTk5EOHY8Ty6XwKi4RHWbc2c3OGiACCA6kGUiFaV4Y51MFHYMC9Y2BM0w0mpqMBgVrNsSJ+jeNiEUXCqXTKEJKFZr6JpA1wSakFy5uDXWqXKxwuBwPzHN4dGP70VoOgsXrxLvNdETeWTMY9fxFIvz1xExi76eDA5RarUm2Z4k5dU65ZkKtmsQyykIZHXWJ7VHQ49I9p4aJBuX2GdWWCp57P9YlHTcYX3ZIxGB9JhqbGavoubfnVabw5vz2m8loS3QuFanoUH0mEHSkWR7kow/coTh4RiNmxUGhg2SI2Am0vjJBH4ys6Wx25YFBAy1oeZCQYIdKH08d5dLuX8JqHAtTNYYmGwHJglYxefDPF1Y5QeFOfLCpbK4n4DruPZXablf52jvQ6wVJtnQYaR3jOvFCqlAcjC/mw88aPHBn9lPbS7Liz9s0ahJLEvi6S4tT9IzFOHkk0l6Bk6gaRqOI6mVJZdebjJ1vkXUhFw0Qz6WI/s+psMFKqj2jmIpXeJaqg7PC1S9R7ugCrF1qTLZtq+6dMu7KIG8LogHqpWhKZWKa6D0fQ0oC7C3w74jUF+FmYtQv9MLEtAzANsPQH5AZepdG4qbyQakYsrKDyqHZEvyySQ0d0H7F1SXVMuGdkadYCMDfT8JoztULUSzDY3fhMH/A/bMozeiNKc/jGtlkcyiyOz3ojo3vIyyEK4AJ1A4jO/SrQztAz6JciSKKHN3DHgifG8FZVHYqNBoAkpLaLGDyPEEQdWEbRMKK+sHobMhICohFVPMIaapoFWGoR5aaME4EkwHopPg/QEUngZeve+he74aAsnmm0jCLvGGwWf27+ZKtco1u3xr3W7bNkKsfy9zxR++ad+4W0ysU+XSB2g+xE2dbbvGOD4wyIXLNwiaDb7ethGuy+GkYK4hb9UAfbI/TrHcZiwfZ3WmRdyMcLrWZv1m7RaipRyObNifjgUfzECgI0lHdD7w0C4uPHuDmh/c1nM3U4CsDhsaJAyBexe68HeSLGqb7gSl+0x4Yjs0KqrWqiZVLssQMD4I0Qi4LgwPqNvYdrtAik4AIUKAaepsS48S1T7B5OSPuBpZYMVRXbzbOFwIJllqrrBNjHA08ggJLws++NjYODSpMM0NWlj0kEcLNaWGga5H0EwNLbKIl5ijXmthSZiuKnZlLZxWHdHR2c8hMqRBqu4bXkjFqSPeP0fjfrIZmz9zJ+QKFG5vABUjeLvPSrosVdMohfc4KsISoCZbnK6jsIRa7h1vdxnlSTfpFuI06BbmdM4rjrIzE3Sdj/da+od0Pv+TWVKJFnMzFskYmDnYuU8j1idZW5DcuChV59Q7nGwR0mzfnIPfKsITK7D3IAQZj7YP9aZLuwy1ksH18wFrbpee9602+J5BOPwElJ6HZhKsg+DuhOHdYE1CeR0OuJcRa5OItk+wx2ejpMb2MVR/j2sodbqug7gpaREQ6xH0u6BVJUEaDu6LvA+j+S6lAy3pjHOn3kKnqzU772dRE6jjAGxwu4aIc4tyUIShEyMFZlJFDl1HUT06nnIuddGFU0XyKlBlNUFupvB6FwbLV8+2u7hZ4fB4vs6pIyWeGIzjpuHlGhQjUXbuH2OpskZrtYHtq6w8QFLAw4MmH+51WHrjJtWmRY/u8mCfwYkT25g4vJ3XLk9RAFZC6z/q+eScNQIXNKGR1uHxoxm+8KlD2IVVRnsG8LQ12u0m4+OS5YLgR6uSi1cr5FJJfu7ULl6/NE2UgL6IZNegTqEKEz1JmkBQs1ku22Q0QVQTFLwAQ9ewnCbRaJRkIgpSkojpjCajzC1VtjZ4BvieS6Vps7q0zND+Y8jcLjZq89y8vEZrFfr3BNTPN8kPF5A5HafmgB/QP7oDe7VOoVhi9YLPwMMJencFtPKQ2y0x2gH5kSi1pTKvPtNE5DTcN9ZoNCXOIvTuE/QNCiKaZNaReBZvTq/Wt+B9uuq63LqHFUBmVxxn5iYFP0t+bBuHt7dYmVqn9/AxblxfZHhga1W5TQP6U5AYgCEHDgm4vgGxQJEgbCrNeVvRQ+3t3zW3LJFoSHpwsUmRIk8vdaLoDNLAw5EJXPpJcYRzTY0d5hoBDmO9hxmTRxmNHOXnPpMmmnGosEg65rL9aAJ3n8dvf/NXePWGT54nGRzcyalH8xw9mSLTq+PaypCw2wFu0yeT9sllK8yt1LkZJDDbknzC4r9h75bG751EcG/cENKByhy0Csp+dUOLJgKk0yqgXypCuaiM+2QWmtXu5zUgG9PxhjxylqLGraEqDRqoKVnug5FjsHQT5q50dcetc9VgxwHlZNzqr4EiuGhVbz+2p181G7xbTcq9SOpoDredww2OgQWyKJEFH5Zs9AlgWxzf0WC9CdXfhOY/BuqgBXjNI7jBIRQsahplLRxBmUfTqMVzE0Wcv4MuONBEWRDHUTyXV+nSzDyFutp8+FoZqBETEfRsHLspkFMWbKxBMgo9fSElZl1R8EVjEE1DPKZYQyIRFX30y8qDXJsH50WwngXnFTCKKqq3BTmD2soq4fOopvOFk4+ysL7C2YWV22yHnkwKq+lQKCjo2NsFqqAb0I0DA8kEExO7efhDH2XHyQf5gFvk1/6//4T68gafe/Qks+sLzF5bvdVk+WLNJuPD9HKdK0uXmOL2uoeO7aYLGEvG2dafx4wY6FYTag2ajkfvzl1kr9WpLa/wSdS8raGCIIc0+EgC2rpgRtx/zL2GhkaAidquj/bAFz8GuPDqS3D6uiATCNJGQE8mrJVCQRl1Q9VjucCOGEwk4WIJfCk527pIb2s7+eQoO60P8L2NHxBQvG1Mi9Qpy+toFNnlP0mKPA4uLZosMMMy68SJkyAeNulTYX6hCXxpY2nT2E6ZjbpkIYBiwC0jsVtzI9jNXoaZwMMjCFtXu7QQt454j+lt5R1/1Y/cLveS3YDbKb1+Inx+ATWJ7nZsR1y41Y3URnm0V1BZkZsoL3gsPGYdpRRPoZTry3Q7fIvw/U6wsBfl9PSH/3cgVdBFxbwbFpWoERqYEfj4x8f53Gc+yhtnn6IQn2P3OBwYy/CRx3dhROMEXpRXL1zmD39rndlrEs+GSBR274Ptu5RjgIRmU3WPLvWArkOjqjF1I+Dyq9C2vHsuwrwxC8/8J3gsoTanMxXYGYAxCf4ViAxCwZEEroOMwtmL8PsVlcE4ikIHjQlgCFojYM+oTpbpJwdgoJ9goUwQqRL33++qly1KBzYVR02CJErLRTf9NVEX2slgbC4sChmlkMqpSPQAIRxKRlUEIwigVFOQLCMCjSK4xfA7TJXxEJ1MSRulCW+vo75viUcF24fT3FysY3uSUtvln/8fzxMHDhvKb1pwXfondvH44VM88+++xbXSpuJpCcuTZaqNMr81CdnRHF94YpCFm0vUV9aZrCzS29fL4JCkUpYsl1sc3D5E1qxRqUmIJ5lZLNIfafPiN17jzLyHrs0Ckr09gj1HNJ54yMQ659Ez2sfrbxQoLS2xYknayx6OFFQaAUGgse4GtGwLqRkM56IUKja6Dn1Jg3Lbw/MELatFOhPH8ySVapsIgmjcvPvgvINoZRfLtkmko5TKZfqjNnpCEDMSJHsM6pU2uZE4E8dNEoHD6uQcjXgM0nGarSatDcHqBYjkIujxGFa5yfFPb6e4sMz8VANdW6BabOCGfJ62l0ILGvhxn/nLEqtpMnbApLKvSbETsnu3EhbUBw2I7RCYSUnv8CBDu/MsLhRIJJO0Ao2k5VIq2qTi9xQ3f5NUY9CThkgC4lFlJ/njEGnA4hwU76HGXAAjpMnQxyzzNO+avjGBHAEp6uTwKOKzgs8sJjnS5GlTJopPgj4+/NB/h70Mhw/3kE2AX6yx+8leBp6M4+cO4K430LQYkT6f4ydGSPQI9oyMs2PnNoyIwG7B7HJAec1h6uYalxfOs1A9TWBfoyWXqUufCHswvM+xXNsqddI7yyZ74G3FNMN9pwVokM5B7yCk0tBswdI1aDchloIdh2Fj6XZHI5IAc79Hu6RoaDvJ3Q7HgA70J+H6GSgsvbnoXwjYtgcmDivDCsIShEDRtG92SkwDerfx7oojDaV/dQf8lsSpCGhIzBMRTMPEWgoUz+7G/wJ7vgLlYVgKQE4gxGeRcgN4QTXF0z4GXgT4MbHMGp6WxasMoWL9ceCnge8BL4MogxxHVa08pAabQVV4mSohjMtocQczPoaRGESkThHP7sEXGu5YhOjofuzpNTWgKQ+0lMIvR5JhZb0OfsdD24DZX4G1b0FjWRX3CdRSyKP2qC1IlG6DYh34wgNHCGKCZ+du3jbXhBAMDA2BL7FtdQM3x+TuJjpqW90J9Ps+p2/O8Nra7/CFGxepzs3ilmqIeIQTDxyi8t1lknR7nF2xlRGfBnar23mb/ZkyDbZlkmzvzWC3HWYLZaTj4roedQmnsoJczOWXnjjCK99YY8gLyCRNTMPAzxgcH/XZaPu8fsNHpN/KWn1rORCJU3XaVFAsVj1RyCUhHYX4E9D2JX8yKTHD+n/PURT3pqZo6zvj1vIgK2F3BF6zoSxrfKv4bfq1LOkgzsOxhzltv8qavF15+sDr9gbrvMgjfBCQlFllmllcfIbJoWNAWEkhhCAQklawRJsliiWPOVfFTDejGxTcS2M/BznGQyEEzEflVGwCLAz00PW4t8jAfTsad3qw96sXNtd56Chno4Zanu9U3yhRJ9yL8mZjKMdiATUZUyiHYy48bgxlDF8KX2sS9sKh6xn3Qsge0g1OB3TtyBHUwK+y9bR/Nq4IKD7/06P80i/9TeKpKOkknHxslemz66xNF8imD9C2bJqtIqMDST71U1FOP2fhWArK1N8D0QRke1UHaTRot2B1VRmzjh3QtkN2kfs8P99SfRvWcjB0KEa6rLP2XJPhXsjsB7sAV3yYW4Q/3FBUdQIVAVkF2gmwHoDJA7D+KTiwLIgdeBA9OoyffB6nbWFeeZeW8/shEbqORgpEEkSH0KOAmmAa3NJ+nYJvUNq5Eb43ogq+ZaCUScKGYaHwjkUNSgFYNUAHvwFBh5Ynp77fq4RKp1Ncfq+wmLeRTAryUcGhAZ3Xlz0utSFt6uwbjTO30uCCB4EMmL1ymZ6ebRiBjwgnTsc8f7EOxo2QmrBY4cJFwSd+8jjJHp0XvnuWtp5h/3iSbNTl+WqLSrPNjbpN0/aJmBafeLCXT358mG/8x6tMt6GFKii80ZQMFn1+8YkITz40wGrNpOQHfHdeZWE21kKHpwWpuIZTd2haPsN9SUwCnLpD3Ydd8SieG1APO4W3mvYtY2al3GIwvTWrZeZ6jUisyfCuCI7psDr9BuuzLXad2sPIWAPN8ognolRmfRanayR3aCxebTL6eILZs6tUTvv4DiQPppl/oUykH6rrkNBjZFJNdFosnPbxU5DYZmCYGmYiiiw6+AUotz1y2QBpKWjZe1LbkEPt+BZUL0iqYxl29CYozs8xeWaNdksnt8ugOnWFRH+GldWtORoVAVVDUTPv3wYlBwZG4do18O/R70sTZw/jDNJHjoDXmaZ1m1aLoPLOio5UMkaLA0AejQCHgDJ5UuzAJMF2UtgLHtsPDpAd1QlcjavXA47ZGrICkZ4o5qEI2BBMBvzsT/41igWfhZuSyoaHY0taGw66lGTNBt+5+vd5o/FDoM1fi+WZyB/lT4J+auXHmXME3qbo43sp92qHazHo265aNLQsVYCaTquA+NI8VJZVfUTfNthxUDUQlb5yNkA5JRPHwLKgMKfWU4IuOKijMlfnFPR28/zsFJwP7YBdR8JeHuGGL321V5XuIEdKZiGeU//fDcJ1L9JuCKQjkVWQa+Fvjuh4yz7ufBmqp8H6/0Hwior67ZoF48/D7H9L4NmodsENIA/BOPA7CF4lEd1FxRqli4m9iEp9f4CoGRBkVnE3fhvFSvURoAci/ZDJw2gOffgniA+DjGmIhGqk6EuIlAR+OYrrRuBEXuHbnEAB5KNAzAetrSisdEPVaBgDsPdvwdgpmPsVmH1VpYAc1H51t0jtvYzdpv/39uf54MMH+dVvfB8ruN2SEIAWSeC5LsLzbpU3diSiQVTTsIOAQCo4cMdwXQbmLJt5bHZadcrONFOFFYqWy8+c6GF19QKXp5dvFY/rQhnjlg9xA05kYKGkkBMx1BD16YLAC3h2ahFLSlICJoQgHhesaoJZK2B58jIP7tvDzk9sJz6ykx3HjjM/tcz2o0dYnD4N107TvFFgqPf+g1IzTpMdKNtzHXCbYS+VmCJGePAQvLwERU9BGB1bBRsDH2JCrachYNSEekXd/s6Ir8s6636dfiLotskhJgCHtTtucgCssE5BmyQa5JhmkhptDDT6ySPDbIZAQ9MMAuFT8WaQnse633Uy2PQ3QZS9HGCC3WgYeARIgjC/XMejjYGJjkHiHr3b+84XbXY4Nr/WcSA2n3Rn0O70FTvHaijnYJY3r5FOIc2dkkYlNR9CTcjnw+/osI1adFvB26h6jhm6abYy3Y6NcZTzUUTZi0N0C6YlSrUcQjkyRboG2P1KzVf9edJ9JTzjPCKeISaqeO06A0canHl5iR8/9xTFep1SycZp+bRtyPUondjfK8CQiDxEBwEHUkkdtygoLnrY9TA17sFAr2qCWnfvnVpyHjjcVtFIs+ywM9CJWZCpQ2oDWmk4/TJcaKnME6h7OBeO8WgLnCuwnof8LrB2SAruGfxSL8baMqnVFpfWInxki+P3vsjmPgNRIK+UgCxxe+v5JN2qL0G3uUuKbjFPAvQYiDYMZBTF8PoG5KdgfATsCFgCgjK3NQTUUyA1CKqojcLiPWtAsLIhWdmokggXn4+i3B3OChZXu2tzdW6Jl5w1VmverQiLH55KC3jJDxMsFnxI07g+uciZyRXQJOWNZSwXKragL28yX6rS9iFiaPz1v/QQHzmZId1wGZ1IIheV4RoQwhbb8IcvtPnYEZNINsWOjMbrpTe7yI12d0CKlSb7tqUZyPTw8vUNrhWbmLogETFo2R7+JtyFBFa2AjECluZ9TNcnP+4i0VhdLtCoBdTqFvVyG19zaVWqlFcC5KiO8HVc32VjtU1l1qdvv0GrrqOnQZoSe0pyLpgnO+YxNi5oL2lIIJWLIDIeyz8u4tug5SE2InCKAVf/NEBEFGRb2wHBKl0L714XdkciYI4LEr06STNB6bUmk789g/ySDVqbqi0JRJSVy23SOzzSORcnsjWK1pdX4OwSHOlTenMoC24VXrgKy/doCPWQZS97SAmThly8yyYVo8tK30b9UgA8gMEePJaxKBLhMSokiJFCLPiIdAMzmsIwBOenG6x+pcAXnt7F+KE4mQkdom0i423iO/sZ6jUwdY/zP1pi8gfPciC+mxM/c5CBjwzjpz7I3/yDH1J0BZXUw5QH/l/0VWMUikWSVDG3QhN3D9KFNLyNCIjmVS+4eEwFuCxXlSUkoqo+w5AwthcGdylHQAqVKd/zoIIvxVKKfXXp5e4P6nRVokfI2n2Hk5FGGU+kIRqHpRsqKJ/IqkaEjqModcvlTacroHdUZT1cb+uOht8hdyqivN2VNpTmkW5BeUp8F0UdI0AchMYvgv9lyOSg9AyIpzBGH8ZvxpBVCawgeYTyxk1kMIAq/J4KB+THQI5A9/D8DnPHnwKnIHsY8nkYiJEcN9HjYPsCPxAkhMRuCAWlbYUXHwHaegibsGBxWXGfp5IK0xb1IGaoaW7oEB2B+Bdg+AEY/l/hzO+C1VKn9W6nnYAPf/SDvLa0xtTaxpveDqRkfmWVI3sm6DPUcMfo9mM5kRTs6YuxUbVJpHVuVH0uVnxaqACwhSqdHwngyqUl9u8YoO1vsHc4xfnnLlIVqkAbVK1AhyygJeHgLoN1LeAbxeBWr7Oq5arJHUra1NGjghk3IJAafU6Ac36Fy5NruHsn+Jmf/Tlq5Tp7zSTCtSg8/zJrZwuMOuDk7l/fpVA2UAcM0WypOqh0Qt3Onj44OgqvzimnvdlQ9Q8bJdU/a0TAoISMp+xSKVRmo0Z3nRdwQN6gjwy95FkPK7Q2czoGSM5zBYmgGYbI06TIhvUZ3fsrcP0adVaot2H5LkEsA0Gf6GFQDtGkFtL3xtDRwwqRGj4ugiRRkhjcG3vDlmo0Og7F5uxE573O/8Gm99/qOwQq67AHBYFqbXq/g3vrSAfC1EIxXZ9HTVobhZCcQ1GsLqHYpoLw4g6gUr4LdBmXQFHg+nTZSuvh73ZsPY0u+3Ur/N7sPYzP3eTx3WAMw/iuNmv136Md6NiWh92SlJsBMgmFwhrxHsj5UHLBcJTyNHQ4enSExco6Rq/L3t0J8pHtCH8Qq93kxrVrXL9Yp1FT+OhoAMIBUYCqvLfsxjLwcgCjG1B7MUAn4NFA0SMur8KUDdfaqhajHn6mU0NdQUFJH3gMTu4GpqC0G/r0dVLrFRJFB7cC/2LW+a/H0QhhJLeIsNOgj4LfSWm1uN3ZMOhOSg1wIJ4HOwUyqY5LGopVqtpSisZLwJwJ+iIM90KlHtbcRFEGpAw3Rx+lqV3e2y5nfSoIdigLGUOnshpQavtcnK0zvWlDqjlQm7s9+d2ZMwZqC+3ADb95rUT9comelMah8SjbE5LlFZ9TDwyxp1fnqz9a5mbNJx3VaC2u8UevXeKJozmWF20GUhpzje5s9IGpFowvtelv+8zV33mm2o5Po+ljeRa9MQ3bDTBMnbHBHFcXNvC3aqXcIUYakhr0pAyMqMnajIfeZ7B68xpGw2Pb7gFWphrYdZDrPu68T6JfkEzq1H2Pnv4ED/70dq7+aJnKqkT0wdiOFNseHODS+UnKBZvUIZNDDw3wyh8sEnhqQIKSIPtwH2uvFJANFMUyECwDETDy4JW5P5KABMROmAweSNCXSdAuWHBYZ+Wsz8pKiZ5+g70PjyBcF7ud5MYbq6xdsTCH7x9KAND21NJ5aR16m4q1LABKjXvzjwQwquX5iZ2fJZcZ5Ls3Ujxbu0EXNQyCOLAPiYekDrcIGa/jsQ5oBMRoM4NEp4rFNPNcv6HzSechRvJJBuJ5ri20qK5e5yPLY/SYgsXr5ylk/jOP/j8/ykM/+yX6xjJ8eCRD+9zrrE6fZ/H5U8S27eNTv/RF/rvly3z3ukYs+Rd56QZsH0zxVz4yTmnZ5oUrZ7c0dvcyNu9423WIpyFmQiKuWKdsBwoF2Cio3hr79ytmVCEUrFMIVU/RNxwWY2vKNtfN7u/VUWpKo+vm3ZnzaofHUUNt6KFounI4Mj1dOtuOxBOQGVC/JwKV9diS3JTKCFiXUPPAvwScBfFBiGyHYAnknyHGv4jY9TeR2hgyK+BCFcRF0C8ghs/BbARkDlUIfhgZLALjII6ArKEwEt8DNnDtgqqmB2AexNMw/kk4kCfSI0hkJam0oOVLqitQnxRIG8CFZhGcBCzpqhDcuwlJqfDV7QBEBkjDRgDJGqSiinUKAa4OYgKG/zk8vgPO/2vFib5FCROd5HM5BgaHee4733nLvj3ltVV6tIB0Jk6i2mBEg1YCRCC4XJfU6y1ygF/3yWndxP/mrU0DiusOc6LCJ54YZ/3CAgueR00qG6vDh9IZ2YoP//aSz8Fcirho0pZ33yuWHZ81p7NfBWSBVB0K1YBqZYZz/+wfMr57gMtzLS5fXsNYbZAJoB2DM9Plu37n20kctT96hAgZByoV6OkJzQsTjm5TmULHhkpVQaeaDWh4sCKVXRl3laO2JwutptKhm9GyZUU1QokKKVLsZIJJJmnTpReu3FHQmyGB5P9s77/DJMnO8070Fy69Le9Nd7V3Mz3d4wEMMDMAOAAJgKAEelFuL2WopXRlLkmJXElLamWXErUUzaUHARIESHg/GG/be1NVXd6m9xn23D9OZFdNT7WraQDavfU+Tz1VlRkZGXEi4pzPvN/7qXh4ONgIBJ7nkBGLLAmLPNDZoVKqSCYMQERVSAdU+uIqU9UTzDSKeCj00EsvPXQQwqWBhoaGgUYQ915Tp5Q7+Hs9WtGX9U7F+mxIi2s6Avxj4NeRRUk6MohcXndwLcpUH7IZjIP0kKeQjIAjwGP+Z84iu0iWkIlM4W+rsdYWfhTppFzyPxNGTpBNZIZD9d/vX3cOlv9dm8Ev/n0VMxEiZ9SplhwKOYdCHQp5KFYUSig0Sp78clfSb5qenMOqFXj+O0s4jkckCVrOZrizguvmaZpVHMtkcMTA8VRKRZer5xyaNTk31Zp3VjjoIell80CHKxeMItC2Cpll+foKXG/Ygz8mrUB/xoEVHawD0MjD1JykmfZdDsNll18vuXyrtlni2XcBLQ06GxmW6JI02OvSXC0FqiBrzobBmlC4DrGQghsQKJ1SaTAmJF3Eqcn3PQFuFK5kIF2EoQGYMkAYoEZlxMbOIK95nXvfSrnhm142HOyDJ+9XKeU7efCZ+/nV//YtSjXZcfVWX9umQtVbkyZdtgQKMGZ4FJdMujt0OvsCLCyUeONkjWk/UlCqOXzmG9M8ti/AmxeW6dg2wn/42V386//8LWoWLJRsAr7qxulFG3PJpuz6dVsqCFWhIxlEM20yNZeGHzl1XMHEUhXXT+kbKthNh/zMHRD/7wLdAxrxRJBIJE0oKYhELcplyOXLeJrO7IUyS1dNvDAyM2XAwENh8qt12voU1GiTWjFL7+4Ec8fyCBP6d8cwNFDiURLbmyh5i2uvLIOt0PtMgKShUSkZaIaOaKlawNoEW/fvrTt1MIKgBBWUNkE8KUhpGvVSjYbiMfxgF4nhGnPHa7TvbGPXcD+ZqWvMNeo4dRu3AIm2zTkaLXhAxnc0hLjzw46i8GC0nwPvP0h61w66jo9w/rNTfL7xTQS+MJzWzvuP/GOMaBfZbIM3rx5j3rzKntQBksowV/PXWOQEJhOYFJDt5lRyVoHGZJnHUw8RV6JoaoRly+TV6QLtgGOHmcsM8bVf+T0e/LPjPLT/QfqiMS7mJzhhX+TEia/x7thDPBr+J+jG36HLMkhH4vztJ9vxFIeZxQWuzp0nEv7uZDRuxYW/DiEb/XmuNN4Dhsxg2KugGNAxgsze+gHclm+uKNc/DkjnYPsDEIjD8hRYjbUAY4stcOOMfuPxtYKDrisLwK2KzLCsR7JHOj+e8IMvm3U0Liiw6oHdCheuAnVZNNcfRkm9D3X024juQXRXR3hgj3vIMNtxcFzsEy54Af/I+5AdGC4DOyHaBfVleaB8HNmZOw98Ean05IF4AyVYIzHWjqNBw1SoLkAzJwsMxFQZ2uNysVh5U+qJug/JxUAvSJmiZBKCZckxC6TA7fBpFlWoqJAKghGUqXARh8jPwoNxOPaL8oHbBKry6Dm4bz+qouOaDTRfJryrLUlbKsrMzCJxBdKlFc4+9x3i8QiKAt1tOo7qcjYjKLFWK1tdJ0SzHiVkXsgEruSqdCwsoroeb1Z1yq5NBPiJthAXajaLpktAUUBRmGl46G7jet+GABvbNq2g2NMRlff3R6nOVXjThA7T4+QLK0y+ssKSI0W9gkgdsLMCjm3CT1P8czWQz0TehZUV6On3yRIOpGIw3AamI1luDRMaTZnZaCXh2oCEBm4QytW3l+Q5wAIWAhimnTF20Uc/ZznNKhnsDWaGZXL0UCZIGBUVgUPTM1limVV/n6s577pDmQzBULdC0/KYaixQE2vHMccMi8zRSZIO4gwxgI5c/Or3umHfetzoXNxuEVmf9WihVY8RBT6ErJX4ceSN2I9Mwy6t++xPI7ML3cgLm/W37fR/BoBvAEPIm/2Qv81lpJGs+fvcC9yP9OLfRD5ke/3tWlUEBvLi20gHJIe8gftuc543gzrpESzU6by/k/4jwxh6VPb2UTUcFOr1DCuZOU6cKrC6Ao4rvePWpDy36OE0QRGwOuMQezrGA4cfJGgIHDIEAgauEiCbWaBQfJPuAbjYFJRmxXUZ/ttBINN3BdZ0N1RvzfbeKJp2FXndZutgPQfDh2TqfeUKdBvw/IslPu3Cl8X/RF3CdSTF1uJ67YWaBkUHJaTj5B3pebYyghpyMBKsrawCvHaBWpGNW1MhsHNgF0E1YHUB3Ia/4AchFwQKEGmHmgOe5a9XNmur9mYX2JuhJg+3rMLliEu5jKQ2XbJB1RA4t+V7r27gG/bENR56sIvSVI6Lkybjlnm94dB1I0WBj7y3nXc/nOSLfz5DOtJg355ufvWfv5+56RW++e2L7Drax59+YZJuXTDcCcdmINuE3qjK7q4Yw92CgY4kX3t+lUtVQVGAqijEIwFqDRPTFdjfJd+1fMpF7GxyqVhl4D4Dw6qxeNEht2JTzkpp4nCPSvKwRjnr4q5AYalBJAVdwxq1msZCQSHYkSK5O0J+ocHUVIZkp0EiHsWuQblusnLGoe2Ajo6N6+m0dYEWV+BDIZZfaKJEDUaOBGiuepSqFlpQkD3t3fZhUtqh86kO9LBDpdLEqCjEI2H0ZJJaMYdZqTE6mKQ+bjF1MkdnyiAUDtO3LY25WqLoCqLxe1PQfFcdzZGP21AkSWwkRWBXgL7EEB8991G+efpFajSRScAe+tjGSF8fyqDKUHCY0zPzzNRXmapPYlMlSB8mRfA/oaEjcFkVF3m+oLFd2Y6mhqjTYESEsRyFrtQo0caPYTT2c/LMKV498x3qzLHMJepUCXseC8dSnJ43mc+38fi+Lrri8MqLn6Gih1gyJ1mofZvR1C/ek7HbFFwpNpEPS8PdNmHxqqQvGSk5zTSakvavadJebUHxnUJF+IGkIOw6BKO74PIpWJ3leh+bWz16rbK2KDJC3WCtZZ3nrk11mgGpbukQua7c96ZzkisCnNYqpUO4G8RxaDfhPgW1oxs1BKEUOKagseSfhXoOmWYYAG/O35mF5EmkgUdknUQiCd5RqLcsCguIg/IQKGek+gczaNZ5YoFhanWFUs5DNBW56WIWrpyC9G4YHIC+pyB0AS5mZMpJODKSUNFl1MsoQuUi1I9B8IMQ3QMxG5yo5OhGFTmyrgXRT8AjZ+H5P9jU0CmawtjIMPfddz/NpsVitowmBKmggm7VqBfq7O2AvrBKvVLm26cu0kiFONwboV43uVgSDAh5XS38OgSxcc+XMmsdw4UDv3uiwv1DKZ45up2JkxdoUx327hpmKNTFuekMw6OjBFeu8ekLVxi31kRuHGQdx3BQrht+CSRhRdbABE2PYt5kynd4uoEuAXVb/u0hswnHgTkLOsJ3bwo7wKAGBQ+aQmY3ZpZhrAShoLzXUSU7z3al4nKtAdUGmOtu9CKSXn9p9eaB+7UAr0WFAnHi7GMPBjDN0tu2N7GZYY5tjKGg4ODRoEKe/PVr0pqbY2EYGVXwhMrMgoO1wfri4rFMgQxFFsiziwa9jHKnT+ymHI07nQxurNu4kXalIe25AFJ1+uPA7yEv2FPIWIHKWuOn3chHfwmZcTiEvGHSSC85hbQHl5AB6weRgnWt7EmJNU7dIvhJdvm59yMF6erIGzHnbx9grdh8c6xlmFJURt+zm4H7HkWPhRFCRSg2rtug1FxhrlBk8lqVYlnSb1xXPkSGJpXvAoBjyEKjkBGlt/cxunruR9UaeEoWvCpNt0KbPsxTzxQIGDXi8QyLnzQRm6jB9ni7V73+mreu4zhcL3tszsPZ/w5DHdDdgGNX4E+ct3aw/b6iRS5uY40G5QAdMuilKNCo+U9YK5MRR65ZHXLs7bjkNCueXIyjBiRUKf4hbDAc+bpZk7r1SkiuUQQlfzO8As00OC0SZs3/+W4mezypoJjzQIgGf/z5b0v+aMjAW8dvvVOU6i7mvMVIWDD4oIo451HwJF+73ICGC0FdIRWEzNUSJ2Zc/tYBnWQ0xa7dMS6dOM7jB9Lse2Q7E+eX+MapGm0GdIQh14TBtjTt7QmWMwtcWTI5URFoCnRGDfo6QpydrYAH2+IGedNGoFG23Ov3ZyIexXZdGvXNR5XLGWgGPHJ2ifyyilP1qNTAK0PbAyrJ/dA1FMJWQohqmXLZoVIUtPeB3fBQy50MPdyNiYO6LcC2o31UagvYloOmFyktNSlPyOyWlRJ4RSChsH1vF00PQprLyotNuh9IMHowSldPG7Pj15g6W6XQD26GNZ5pK1Xrh/dCgyqxMQ133qZrt6AxZ5JbCLDrPb3UlufxhCCeasfQLcYeTzB+OYPlenSE4lydWCKY1OiOCnKVeyF1dfcw0VhxHMxynlCljWajgpM20RSNhDqC52nMu2/yyeO/wsErP0pbaJhcXSDcCI+l9pPufRgtpFN1TCbys4zX5ijUG7he2Rd5a5DjPBWxQtwdoIMO5po1uhSdQmOOC96XOap8iCfFj1AmR5FVCsziMU4PWdymSsjs4Kn7+xndoREOq3xk4OMII8x3nl9kZWKWqcJXgY98X8YPpOBEbln2YqmvyPkr2SkNecuGYtkP8kX8Fg3rsgyK4hs7iqzpCgSk6uHuw9DWA4uTUMn7RtRNkEISj4aRwbsV5FSb4a0ZVD0sAzOm5We9xN07ptfR4vLqCnQZMHYIjH3yBCz5zIg+hWZN4JYUySs2y1CaBX0nJPdA8Y9l4zx+HAJP+n0thuREnuqEwCMw3SIPt8uzEXvBGAJLxumd2iyrWUFAA+qKPOGaC9eOSWckV4B6CEaTMHQE2upw5izk50HU5ERai8pOrokIdCqQ+0ew9AREHpdNSZJJaAalsWBmIF+DYBtENmXOMdif5OkH9zGcjGB6cZ44uo+L4xNEwgpd7V2cPDdD04J0T4BrFZdgKEiXgGzRIhrWCccUZqoO7UKe7uJtruGwJteJCtI5mZ0tUsmeY28cVM3gtdUyT77/EXYnU0RFgItn8xSQS3cVCBoKDVfQa+gMGwLRdOkERgNwsEfj2UWXCQcmcjIL0LL3WstsGchrUO+KsD3s8eFQQBqOd4kffRSiCbg8Dp+dlDbTxCocLsh+ZJ4na/sDvvpkrQ7VGiwXpIJWy45q2adwe9t6lkU8FHayDROTwnUy+9uRIU8fTVJ0YGExzgT5DeQsa024cFWAcHBuE/B0ERSo8jqnMDhP6A6t4rtSndrI27rRibhZtsNjjRrf+l9hTW1KR1KojiPTWX1IR2KctVqMV5EyaQK/AQwyajLpf6Yf6TyYSG2IK/5+DdbaHywib9Yya4oI5/3vOoJ0LpKsFYm3ZHC3c3O53tvhSk+cqtqkc+F1kukYsVgQQ9NQ1CA2DYRPYknFZQqrUZaZUkuVdEzLgJoqz+Gv/fgzHDn6IJ6eoeksILwSCkUQKwSw6e2rUqll2X6/Se/rUDi9yYO+BVoPh0DaO0cN6HIgMQXjM3BFgZPuO1ZovXdQ8AXkkbNOS0FKQLjLT+8bIFpStq26jCjggR30d5GGaEAGmxICsiUwQ7IRkd2EYFRmgdwkJKIyalGryQmnHpQ1IOkmZOt+EO17ZMvdyHv2gOwNTsYd8b+BugvfuJDj4UGF7qBKOgD39UJfl0Yu7/L5s6AZCuVMmfGTNqmQyg9++DG69hzFHb9M/loJB40d+SxJP9j36irYQl6SzHKB6ZUikbDL1aKMTiGgUbNRNQjrChVLUHRcDmzvwLE0riwXUYVDZ0qjqycBepKLpy+TiG5uvFQD9j3VRa5SYeHZBm6d6y1gm0VoHwlQuNpgYF8nK1YVdbtGvEclFAmTigVI7IqjaDUUu0zXA73Yik71ddDSOvGIRqXpEWyzGHhvHx1DUWzXxFYU1ECAQMNk8tUyRjjA6I44umhSXc4wfqLGwmuevJbrJ11t3f8hCHSrdPVEKSx4dA910N+VYGrBYmlikqlvFSEOQ7sFsfsShMwiI0GFzs4A6b5uOvM1zp7I4dahmtbeNi7vBHd6f1l4/HbhRcb/8B/ySz/yM8wVV/nizOcI6ik+cf8fo+U7WDJPslD7TywG/99crvSQr7XzUNvfY6zvYRJxHTSFUFuI+9VDLGT28drpVV7Nfh2beWxqCCqYxKjQR5VtWAxiE+FE4xdoD4VY0R8j7nYR9hQMorQpbbSFLZS2JnabQiL6HEbwo2Tm4gQjkM8FmZmvM7StjY+Gf5VnT33rno7dXUOAXZA/mi9mYdsy26wg6Rtl3z6JRuRvTZPN9ABZL62uOSCKgFAItu+Coe1w/g1Z6L0RWkqrUWTAr5XJiLJGFbl+HzhQLoIakBK3AWMtY3LXOAgEhJzcNM3vhGbI56MOtCt4TYFXBYoCKiW4ehZSuyC5C7X/JN6rDjgfhtRPQmIMCjlpSZZcaAgIjYC611dmaAN1j+SlqReQ8fEmCA9blyqOwhZQcVCny3jlcaSFosquidMmNNuhNwpHj0KmH+YvST61nZMTYj0EkfdBXxcs/m+Q/2MoRkFPQTAltdLJQXV2rVHKJtAdT3D1wnkG022EIkk8y8EwAnSn02zfvpNUOIKyMkNlqcxoMMREzcTM2xze3kViOMaeWII3jk+yMFfiYHcbKwtZOW+zxlxp/e0CJU/aBt3IJTANOI7F7l0JUpE0hhvBPvEc1y4usdCwybniukJpvwqdsSBdkRiDg90EVIflY1fJ2oKyBZ+ddVlhrSN3y8acQi7/vUBJgdGEpKFNF6vMLpZZcuFf3eW4PfSQHPbZ+bWlfLYBC8uyPkNRZCZD02SRuO1IJ3/V5I6ZJjfCA+ZYZJlVVBQ/x7sxXFzyFEiR5hpXmGFhw+8UQjpEdwsHm8odcr7vWt62hVuleJR127RuNHXd++t/txiRAmnM/59ISpOO7M05ioyEnEY6B7+IfFyzyNoB3f/djoycnGetYLmKrN24BNfVDPA/09KOBpm6mkTOVX2sdRGvIidLxX99203O+Xb42A8dJaBFJQ9WkdkK4Ugira55DLX3oI5UWVhyqJZhpQ62B01Hiio0GrLZVdMET52i5n4NtzyJG62iairCLeM4Kq4XxyOF56kUV5dZmnlnxH//cK9rZa+/B1qRgkeicH8M/nQVvoXk9N/oFCtsKmBwb6AgZ5gIcqYSXCcPqyno6FJwmipF20UNyLoKJSizSXpcZi+iMTlhJHSo16UccNF3FhxbUt2MoIwauh6IulSh0h1wVKjnJBWuqUC8CJ1ByDTXOQB3JCdzbxDUISxkqnb9V95u0gsCHUFYMmHZha9OC7bnXSIBlclpj4TlYlZA87MbJyZcyhX42KMhgoaOZdXRIzq9gyH0YIBKbokAJipraeQmkNU9kobKzqF2Vso58v4YuQK0gMJ9XSHOTUmlpKITIZYIEa3YRGnyyKDK40/v59UXp6hFIijqnVQovR3tfRGGDuxm6RuncSuNtxSVKRlYOt/EaoCtZkhsS6KIGl69SXsyRWk2Q664gO4G6enpIhCJMn9qnOj2NpYXixjRboYfThFfybP3cBeGHsKzNVaWZ0m399GsNahWZ2nbGSPZHsFQw1w+NcfSNRcR4K1alP64GT0KKAJ7FsrXHFZ66/Ru09FNBSECUM+xONnEaQi8osJivIgRMdk9EKQwVaHRv8pMJYiiCqquR2IwwO7eyKbG7mZoPXZwa5agh0fVK/DiwgXEZz5DWVvianYaXRV87vQv81jbv2RP1xO0a22U4v+U8uB5QnmLNxfe5MK5D7On80NY7hIDsaNU6x5XVzMsWKtAFY0+XBq4ZJFLfZEyx6gzTkWMMsLHWWl+i5P8IXXl7xJTOtnR08b2Q8N0bN9JLaRiGxr1ks3EyRmuXS4SV/pJ9QzQq19APf0lqumf56nHP3hPx25T8O8N15GNQBXPp/3r0h63G9LgA6lOFQrJuoxWMNBQ/XXcjwgGfCqprq9zSDaAi1QxXEVGoMPAdt+JuNyQ67Div66ZUro0FJcOxqadjNYX5xXp6QjkfK8hrUpNyC+0FKgJyBXh6n+ExIdg9DBK2CRklqkHnoCh/4zS00EgEMbMOLI5R7vuGwIaBA9CcxtE28AL+apHHwGtCOKz0DYs7R7dw8udANcjuGsfjend0vOjIQ+2EYG5glzY2+LQOyAligoZyE5A5qLkt9UqsLQNup+ExjekE2Ll7qz48g5x8doK4WCA5gsv89C+3XQkk1SqDcKhNMXlAmJ5hdmZMk1b0BX2GFJV9FSYomJx9vI0ddNgIdMkHAxz396DXMy+SqbZvK7s2RINaAWXK2JN8hZ8OVsLfv8VSawKCGj3fFEeVWEgEeWprn7Cne3sOLCNaLyPULNBMmazPHWexPFxriBp4gZyrSqzRtsKIW08xz+GRQGTBQ+vULilaNHtYASkbVY2176r4EGhIhNVAV0y22wPqnXJVMlmYdF86/e1nrk7ZU8LBNYdGvgLzNOkzCqFzdMSb3ocd447djRaDkNrzV3vTLDu7/XUqBvfX79da64K+L9bC9FjwCPAMST1aRxZ4D2OLBxXeatUbQHpSNwHTCCdjtZ3dCMzE0X/tRhSGvcg8AXWmDQHkY5FGeldB1hTLlWQTsyg/95mIMwslrKIoekoiourGCiEcVwHy2tgOSVsVeAYsFqC2by8MS3TLx5yZBY2FgfLvUS14RIyujDY6Ru6Jp5QqVsFbKvGxTM1/uz3nbfICN4JWiyMAPKBDSGvdZ21wq5WvUYUOBKC4RB8MgvLN7nrwkjHcGQT4/aOEUY6GZ2ACkoEqbGuykhfPAFYAiPiEqpA1ZXORygMqgmaXytTLUslr2IJsCHgyEXR0KSTgQ6JhJRwVJBdkOu2XJAVT0pFOnnAkcG0g3HppGRaKZ8NFthEQmZDNl0ceRPYPi3vbicdCzmxBh2fHhWCQAw008MLqFzOetimlIOvNzzyDXjvYIhdg0ESiTT5pSWmL54hmQ7R0TOIaWVJpxVCfsF5C1UTBmOCaqnxtnYi8xmLSgkMVcEQcHlyjkgwiC08FkyLxQJMLb3Gj7x7jMysSsbdXEpj39NJ5i/NUG7WUNsVom0alSsO4ZTCg399B45p88aXpsmU6zz1s9sJh12yc8vU6y6Rrk6alRyz50vUnAY7g11YAY+l6TyFuSaGkadjuJ3sqsGl56+x+/ExDBtWLi6iIYgmAgTw6BnWiIbDaKbF9KUargfKiAKLAlED4hBKa6QPRggoLoqtsVhr0HMkRrRfIT9VIhFeJtEVo9xwURIaqSEFkYgzdCBO/toqjW0DlLw8laUmo702jcUiIRf0kkOi+16rE9zZQmqQJsoYirifrywOEaSERpkaRRyaPLf6KZLaP0azx2iWnkHt/jTxWJmO3jgXZp7lKwufpzu4g/OLWVT6aRDFJoGqJomq4DouqApx3SGklCmaGdoVnYSioXphjvCTbE93M7R/L2/M5Hhloc7ESxGMVz3s6AqBhM7BHTvoCPej9sDkosmV2VVGtFUedD7DyNgHuJoZvedjd1vczFpqSkPHbUo7ORKCsCGzFI2GX5ORlnNV60dT1owf4S/mwpMRWseRwZWboUUDcZEGXxjo0aA3BlN+QXk7clrGhdI8hJJrzstmnY3UTimd65lIwnxAgQQoRSGzxyGkhWsCqzOw9Ck4/Ch0HUDkFerL2+DQb0Gyl1ACXFtBNVLowsMNg1vToKBIzflmEgKaHKisALsT+ADoizB8GOGC0ASa8hzOsfPYD/17ubAwj2zw54HokpNxsSIvRKEMY0OyaCXQCW29MH9G8ivtMuSfgfQJWF3Z3ADdArFEklQqyYkrkyys5Hjfow/Q0d1Ow3Spax7Vzgilqk7ECFGIpnG1IMHqMpcuZ6l5YHvS6/HcJjXb5cjOYc7NTlFtWKxnYK5nt4C0NdL4rQYUsIJhEsEoS8UcJQQ7IxHe/Z73sHfvYfYeOEzXYBpPa1KfuMy3/utvcmFyhlebNuN+bU+JNduzVasBsn53EnmPdSZVVkoejvf247pbWE3IZ2Bu+a0B9GvL0JFa609jWVAqy9qooiUD4UkNesMQCUh7q1mFqxZ3WFoNBgou4rbxSRuH5beE2b8/uCtS383oUTe+t1H240bq1EafW/93B9JJaAceZ00mTUM6AL7dxoz/9zngpP8dAaRB/HeRN9hp/3MfAD6KLBYv+NsEkFkTG5m1SCOToC1106D/ug63YMPdGqoWQBEGplfDtBdxRB1NMWjaFrmc4NRph+NvwOocDO+FgiWbt7ianGdUAWENRmPQY3YRaTyO0Ou4zQpVr4TlmJh2lcxqkVeeX+D5b9YpLN/ZsSn+OUZYWxhCSCWlziAEOxSurgiK9lpX2BCwNySzu5/K3Vy+O4107MYU2fXye4rWLNYl/1ZSyHqVBGgJiEdBd6FakHUwmgCaMs3p1eSYC3/VDPiedVjIAkbPA1MBQ5Ep8nAIIilZBKcIcHQ/2+FJnrOqIge2IKOKhYakWKnazbnOlQqb1pS/FTwhdclvhAJ0+XUSzk2+N9apodVcplbAdhXS7WEKKw2iSQ09ojM9aVFFnqoJ5KtNqo0Q2tA+2sf2YrSnUdwSyXg3oeij9Oxe4ZXJL/Lq9Nr0WnfgSkYQCjRp3jA2gYCGEVGIA6qj4AQEzZrFux7ZxfmLk7QlXcKROF99c57X5qoUNumkNZoZzn7NwWmHrr44fQ/FOLu4xN73tTGwfxfxgKBeLHDmuQJuFtxewepEkWvH8xx67zZ27O/k2remmbTKFDLSScjNuygNEHs16lWb6VeyuA3o3RsnYKwiggFOvrBEW7dGJK6RUqssHZtA7wzQnBFgKmi+0IwSV1Ajgp5Bnb0Hupg4vczA4VE6dlVJBGwW53MUr3qcL9dIROvo7QYh06VagrY+h+1D7SxeXmV+tYp2IEq+WqP5/FVQBNUqlLIe7bE7XfLeOTRChNhFgxQqvdikqdLEYxqXfjyS/vpRJ+9NOEiu6AAAUchJREFU8Pzy59kf+CCN5tP0N/ezO90g3LWD3fuX+NSF/4PHt38CXT2Cq4YIJ6PEE1FUS8XOQCIi6GgLs6s/SAKF5786TzKqkErEOH36Gnm3ijLcwe+dfZGFUgFBiLZaP9vUbTySPMDRp+IEYxpuU2El7pAp1HFdl1i4m+OrXeTe/BkuWo/xX/nL79n46QGpKOX4okNve3wtKUCBLoNXekA6G6oneeOuDe0dcu7TNTkvKf5ivT5g6Hl3Hvjw8JULkTUYricTDkLItTWIdEZqOShnZSNsseHB3xkcxa/vqCFTpGkPmtr1OR8hnyFcD0rLsh7CFFAIQrYBo0cgGECPKGhBOZZaSCMQVLEMD9EH3mIJOhJSy7yGDFcHLbAW5EId/RmItUNIwdFUjPBj0MjgnLvkR6Q6/DT2K3IE9N2gNSFaA/fX4Oxu6P9x6B6E8Chsj8s26jUPGuMQOQBa9p5Hn1YyWcbGtrO8OEulVuXa0gpPf+Apzl+4TCQQonp2FbsR5MJSGVWtkki1UyrXaDrSbuiOyELi5YbLycwMh/f0g7PKxKwFFqRjBosND9MvABDAkAoP6QoLtmCkP8no2Bjx3Q+h6+18/U/+O+FmFQuFsdkFnIU5Lj33eU6ZDbLZLOPlOmcaDotItouCDCK34nYaEFelamcAaI/pZC1B0XJZslUMVeCsKwZqBVjvFpUqnL4sy3birPXanWvC3qovbuBCvQGFOlR8tak9MRjrg+E+GexUkSy9yCScWHlrofjNYN/z/MR3F++oRqOV2dhs6unG/Qqk89ByIMLIG+gy8oZ5AElf2od0Ht5A0qwqyIscQmYx2vz9/QTws/7f24AfQDouv8Eap24J+Coy6B1F3igr+A1UkHNUxD+mO7Td34Zas4jrFhCiTNNUWMkqXJn0OHna5dwZj+yiLDTFktzZ4UOg+aoFQoOeIDycgoMdELEWKayeIpAYIFOZZam4zOWJPBdP1Zi66lAqSGO4czsISyE7Lza8OAoQicDwgKQEKTUZvY8EoKsNdvfAjl0B4kNtXL6c58xxi5WcQmZF0JeGhSqcr9yc8dOBvE7twIABD9xbJsbNYfhfGgGlDUQS8DMTeKAk5EJr1taUMZp12UAn7EcXFFUqJ2ma7O4thLwWIc13JFSud8QNBiCUWNuXroIlaeI0y/KzIR2IgxUHsjCdg460RtTwqJQ2fnK+G07Gjehph0ZNRlp0JAvAEBuLGgngyrRLRwwiBpgeJBNRgrpGPlPDDauEIkBVPkNJBYYi0Dm6nfTIwwTCA3SNdKA+ZNNcmCQQ76Jn907ef/8bHJ+Zwlp3vlUPqs2N7iwPVyg44QDCdEgAfakg5YUstYpLACjoNcnN1WVGajNQkxp6h4NXgrYRC9XOEEopBPvjnDt+ioWZPG29OskDOovz07hll6mMhx2Qz4MejBMaVKirglLOI90vs2TCBM2NEu1qJ7FzBbuuo0YcAigEYgaBuAsiRDhQZvWUx9x8g+TDBgSg63AH246kmDk7STQSob1dkEiHCUXD9O/uIhhOEWyaOJaHmxU0MoAOwlE4uKeNssiiNqF2qcqsdo1YUGPbjh5mLzZZfraM4sHgU0GEYxLvUunc+72KygdIaz+E5UXxxCImBQIc4IB2mJ5gO0FLp+GUuMTrzPFtBItMuZ/HbnSQII7WiLIttJNoqZ3OgT38xM4gy/YSHzxygEzFYWapydRshf7+EE893UE6EkBrCrpjCinPYSVWwtMClE2F085Jrjh5wuc6GY4P8sDoATpTKaKhMNFUkK50mFi7SnFeUPEbXA70xClOFrmYL/KBPf+eoz/skAvcjFh8D+EXNyqKnMfrdVnY7dq3yBw5UmbTqgEGhFXwbNnjpFkFbwj0Nn/e8x8/zadQiXX/h4I3+wKJlj3gItflnABRXFuGWiVPKpLaZZch4NMH3JstKLeBa3qodQXPUiCvwqyQBkEMaQVayElV9cD0Q4m1Aiy50BGFuDSFgiFB01bk2qsKVE3BQMXohJruSksxpsr91g2pS+psg+Zr0HYICKC4AmxQzF450LlXkJyLCPBZ4POgHITQDhmFCnbLxieT/zuMfwYKH4aOj0F6FLrHpJNSd6DyUYifgOJdUhVuA8/zWFyYJxEL0GjCiQvjNDwHT9FIRVLM5aq0d6aYK1aoOx71bAaArpDCjqBCMqzS1BRWFlyuXpmhq7oMOZMuFCwEH9o/Sj25jT/51rewXZfOAPy1gRDNJZvu7iijDz/BAw89RrG4SEdnD2e6erCXppiq1/nOhbPMylH0RaplHe8kby2gbgWGQa5fWZ8arAG7+lO8u7OLb795lZWqXN2ihkJXUCeOx5Cu0q/dfSH9wiK8MLUW9G4xoCuupFY3XV/MqA7zpgzMduuwowe2jUB3ByTi0mYY7oLBfhg6D1+6LH3L/yfhrkb3RmfjxgxHizbFutfWF31vtK8bPz+PLOKOI5UrosjgwW5kD41WEXgQOY9k/J+HkM6JjpRd/a/+PpaRxuPH/P39IWsZjhBy0ptH8gWz/n5bagiteUogi4kydzRKb8f80jS5ksbFqw4nT3nMrQhKJTAbMrCCy/UagoUpKCxCugvau+ChYXgkLSPnLxchVxXUz54lUzpGyWySyUJhQXrPkSTsPaBy+HA7e3f2g2XwuU+f59TLjbekuyMBGNsGH/iIwu5dUUKhXhxHo5BdIkadlCqIpuJo4QiKXedAm87YwQSNUoyXPz3DVyYEc82bO5dxJB0t7o/fqA37NlmYe0dohSMSyAxGFDDASEqnQGiS+qQEfQk8n84UD8jCe6FAOiwjeQHk9q4iqVH4NAPFk4Immgai4Xf6duTCLoQcf9WAhiW/K6JAsyFFTwiACIIXAUeTn8sWXLQg37X6jEBYat/fCmoQQraMtAjkWnYr2B4s+YRbFcHibJaO/hixniQNq0lbUnDYchmvSnpVw1PoGx5FVSt4Xhar3MRsRIkke9H0AJpQeP/TO7kwtcJnT9dvG6zQNWhrjxANG3hWk8qKyUzFhFyToCFIorK4WqErHiYRVhnr21xB8+nPm5gZ6Dio07E/xdLJLMmRKHUsrkxmsYXOQFs3R0Y8VucWmF0y0YVK8GAHte4wl2eLtO/vJFhrsrpQI3vehTpse6iP/r440WqVox8dYG6miFXJM7xniFqpxOzZEmbeIf1YksblMsWaS7DhgQPxsEvIq4IqcB2TUt5DOEHauwTtvT3Yjs7sq6soMYcDj4yycOYSTQ16d7XR0ZOilqlTmqzRWPI4PlWkc2cA5QmVYDpGaCiLJQyUzhQDVpHunW3sPnRgU2N3t9AIg9uPzjBp7qPAi1S5zCW3Sq6xgwHRQxcaaYIs+s2nPJrM8yohenAZoa8UZKhmUp2rk9Pq2LrOpxf+knD7IHt3beenn+ojFgpi1h1KC3Wi9RqOUgCriDv13xDaQ8xpjzHu5ChQZci4j/sHdrF9JEEiKWX/KqbHlUyeS9MwkkqjoRFJKvQM6hjBQabndV69eJGQ28eP/87B7+qY6QaIAAgbcEA40L9bsnAqq8gF8xYQQNFe60AsLKgtwaJkldGpyd9o0mEHED4NFCGDYLdCB2uNPovIAvDhuMaYC6sVF6P1vcj1tl6DdBBQNleUCtAoNqSMVYefhskoa4Xg8zakdTkp1x1wLgMmFJfkF6clDUqLCUiCW1IQQoCrYBbkvBdMK9Cfgum67NptKNLhEAoow1BKQzUPr7yEeM8j0GHgWR7SqhgCEqD+IXifQfInToFyyK+4N0AflAPsXoLVS7D6W7LRYPtHoPcZiPRA8MOQnIDp3910z4yNENAUJqfnCAVUQrpKWBWcPj9Nb18XXrWCMCvkZ3K0K7CqQDysc6hNEFc8PARWBOqKRlCx6QcMU+O5iqCGvARvnJ7isR0N3rennWMTed4/2sfrq8usNlz+VjBIf7PBmc/9OSevXuLQnt0MhBR+s9qkirS7mqw5DbDxktky9Fu1Dq33XeDitRz7MzXGhOz1EQ7AgyMBeoJRLi9VmC87TFh3TxU9eQZm6terbq6vX65fp2HU5fGUalL1MYQMQGpCOtgOMqgZDIMWkU0t3xOQ2bmvXJJ9G+8VNBW6uiGfkxnG7zXu2NG4GV0K3l7Y0oK37j1xw3utTAjr3ge/eAxJZ1KRF/EIkgXT+ryLzEAYyILxXUj7cjsyKfkIMnvx58ibdNTfxzeBzyCDG51Ix6NVPO4hZWzXI+L/1JGNTjfL/okHfpRkT4Jm801eOP4mJcsvdPK1fT1/MERTHky9DKFleHAAnojD6SX4ZgGsNERTJrpqEmwL0K1EKRZrJDvkDdzZBR/+gQEeOfQMqupQqo/zwR8JUsw2GD8nj6UtBdtG4Wc/EWJkRy/B5AAEErh2kf5wkWazRjDSi6KF8VQHEbAJWSmCIkbp1AJfmxTM3kY5dJt/PRrIhyalSmrRdwWtHhcG0CPHUw3I+d+pIqN+SOM/oENQlfxNTQHDltkfVZVrlKZKZQhNkQt5KCKLJAGcOr7OsKRG4UiabiEnG1GFQjIboChyglAtCFXBDclFWpj+8cSAknQ2DE0WQza+Cw9+/6B0Wm81fy4ucf3BWz+nbZShbA9ANKBQqslo8DMPBrl2tcn4VIPe4XY6OuLkV7LUmnUMV9KhF20N22xSnz5Oo15h8tRlIokEne1txDoDOIpLrlwgatjXBQduBgWIhgxWV0yEV8cTCq4r8DzYNdqFV8wzm7cpOFCpNdnRH+OHPrZ/EyMHjVkw2iDQDkY8iWMWGD7ai6s7CN3FzblQc1BSgmodYh1x+ne20Tawm7OnruCUm/SMdBE3cuhukJnxDFpFwYgLCvlpkr0p0sKlErAIx0LUyg1WV2uYioaTdcmVFbLLHp3vTaGZJp5o4BhFaqugCpVET5KZlwswZjN/dpVApIKaDlOt1AnYCqrqkN4VIDrYyc6BIAtXpiktQn1Zxrr1gEL/4TQNy0F4Ng/+UD9LjU527tlL1/3TqEod87vVpOQ6NBRc4jgkMIlwhAABAmqIkjeBiWBF1JjjLBoZPM7gXZ+tV/BwqdPDLCukvBrLloNDmbrjsmfwAT786KN0ticQwmPxSpNXr2SYW5ghIk7zrvgiHbu6mSlVudp8ngEvwhI72MEAfbxM3D5PppJm8tgq2WaDUCRAR7KHcytZ2jSFY6VZ+sKd9HbGaVou2XKTSr3COFNYk5M0/9EK/+L0j3zXRs7xHQz8+SYYgfYOhWhSMC/AVMG8Dc+3lXFY/0J9FRZUmRlpS/s1GS5SOMOVjB3bhnibzPR6N3lg18c34si1oFp1mXUl26CBXJdbAUiv4teAvBOhMz0C3Uqrxl9OxqkaRBuohxN4uiJt/owJzlkIPwhDH0cW7gCqIJZGrh+OzGDrKYGLgpeBRl5AWoW5kHRc+v30N6pUAom0S65zqQH5Oko6iRqYRLpzUeACiL/kOtFYvAbOJ2TEyxBgJ3xDoDXzVsA6D0sXYPm/gt4HscdgJCsvwj2E5RczREM6D+1JI5oNTszWWV1c5H1P7acRDjI7PkPOU9g9lsDGo1FuUmx41CxB2BMQ8IgqMGkLxpdq19cTBzjesKlcmOO9gwYHDnXTDHTyhfEFmsCvT2Rom/wmtpDUp2dfPklIWbPJakg/0BBv7+sUAnrDKrMN7y3OxY3I24ITedlHWwciFnxp3KRdMXESGqaHpHXcJc4vSSnnNh06QjIYXLSkzH25Ji9tyZd8ryHtzpoLc6tSkTixAmPDMDAgMxuhgHzunjgk339++t7FIIe3B3joXTpvvNygkIVCYWOmy3cLd606tVGNxc22W5/JEDds06rbWP95FWkrdrNm7IT9HwU5f8wi07EF5KT1UeTENYLsmfFV4OeQBeLXgL+B7M3x75CZjhbzOI+kTyW4ee2F4m+/iN/o6Cbb3Q5DvQcoNafoGjC5//4YFlVsU06uju3fTAE/Ku7Adgt+eBh64vBSCV4pQECFw48ZJAwXs6zz7iPvJ6T28oXn/oqLl7M4FrT3QF9fHAKz1K15mrbFzCWHzII8jnQ79A3Bww+ojA5sI9H+AKrh0PCWaBYu4boF1Hg7hhZF08N4QsNzoyhKkOr0HL//1TzTtxkEDfnwGchrFExCLQKFneqmi+lv+kUaciULID3CVvStKRdC3ZU1ga6/eIYMWdCMJ2kDEWQ9gh6TMo+qIZW9TFtyJ8MRXyZPkRFAQ5FCIDqAKhdapSlTpbYGXUnprOBCxIFAEZYDsijNaRUYWVx/GEwT4p0GSta+51SppUkps35L3OQ7h3V5TjmxFiUaTirsGNB5+YJNTBEEGi61OlzNOYyvrtAVNahbDqV1TlPdgnrVInPtNOWlJUorBfrH3o3nmViKghZOcun8MoW8jaHcvDakdaj5ko2hKjQdj0hYZddImMmZJjRNLudtUvEwT+wb4sc+ej8HhlViyc3dcaEOhT1PhahZTa4+P0ljUeV9e/ZAADIzVeYvFNixaxhdy1BIqJCIYStQruTAqlFbzrGq2ezc000sbrJ4MovVEJz/3BJtOwOk3xOhniuRbWr0peqUDQVH9/BKAq8imH85j5qAUEzQzKp4FWiu6nQe6kRoeRamc+x4aoDtwwEufGmJZE8II1an95FeRK1JadHFrtnYBQvjviF0U9Df41EtLFFasNh+tJ2uPo3i/DyhqIWuBhhIQf7Ea+htAcIxk2r5yqbG7s4he5/EifDuiEpHm8dMOUmo8V7mxSEqYgGNOjWmaHAWma9uhX50VCw8JqkxzynmCGASoU6QOMPlUepLdc5fXWUlu0ijPssr2Te4Zr1JlClitRS56i6y9SwZ0WRIcRnVk7j2GeDb7NWLWOYeVtMDHOnawfSKjZcU/N3H9zH1SoE/y/4mJ+0K4fJeAqQBhSazZHiBWWeZ42dq/Au+e44GcP3ZFUIyaYJTYIUhkQY1BqtL0Mxzd8aEgNoqzGhgD0Fnuy+JK/wosSuzvok2SHVA/iZc4hZXviV1WwWuuZBMBhmOxXhpPvcWA0q1pDy4Ebl1f45bH7sj+SlzFogIDNdBzMK2XvTRIFZLaMyryxTOwM/Dzu3Xm2XpcVAtQaWuyHVYBzRVStQ6ivQXOoHDOlxxYcb1KQ+KzJQoDtjPQ+n34cKPI8Y+gRrtRxJ+POCLPn2hhVkIlCVXLeTJBWLDRUDI4kL7KhSuQvMWhX2bhW9sVasWb5xZ5fCuYUY7k0xNLXD5pfO4AQMvaGA3HM7OVCiZHo6AjlCAVDzKwnKB/qSCwxo5Y/2ZCmDShfZZl0dEkxlvGtfzcJA0dYQUjGnzR2pOvNXAHoprBARkSu71gLWHvL/yTY+eEFRsqaQI0BdUEB4s2Wvj2TqeDqRNVxXyxy656AKSjbsf0444vGsIujqlr+m4UKtCqSqdiems7Cs20AN9CzCZl5mNcl3206iuQM8UPHMAdu6QvTd0BZJxeHQMLq3A0maNznVQFAiFVK5echnZFuTwgxGe+0aefOZ752vcvRu3AdbTn1oOwo00K3hrTQfrfrcKxTdyRloOiVyUZBJyBekcNJGF2nv9v38BeB9yPvgUMng8iZxLMrxVEc71/z+ywXst1Fi7QUNsrmAIYKH5F+Ryy8wv1glrNgd3qKBp2K4MGUVCQTr7wox09RNemCUxV0TocEKBE1noHlV57+FhoolOLkye5uDe+xnueQyVID/24QhXDr7IcmmKaMpicFAnHNDROEqhuMjx18epVaG7C4YPwIFBGOmNIlIDKAY03BWKS6dwlCrRcA9hPU2YKJ7WTtN1UFSLZjbP7382yytLtz9XP+iPhiyVUA3Zf2g55G1aHvgtaOnXheB6fwxDfo/mhyxUVT6wkZivPqHKHhcRxacUNf1mkS4otpRXVDUp3RjUwKzLLIQegKCQNAI7INP6uiLvnYAiaVl6VjZTrKqSa6kbvnxyDBqdsujSjPpGf5O3yUpUsn50ylAkiROZedmoO+fdoDMIc5us511y/A7NYTiwN02hbnNussr5nI3tQdaCiddsar4BgoDF6tujbIWGw4XJazzw1BGGHnk/+4P9qEaI8Te+TmahwGjHIKVqjEvzd1YA5wmB7Qk6YgoP7EuQqxgI0SSRjvBLP3SAH/rEh+lIdjB9/GUuvPAckWQH+z9w9+e//+PdtPcFyb05z8hoBwtOiUStwvCuXVQOj+LZDVRyVGqwuuTgzK8gjABuKENtsUlnu8bKeInpkEZbLEZfTx/Tiwvs+/gQhx7cTcSwmL42w5UXFigUpkhGDEzXxJ2Xi51Xg8GHwzz42G6W+vJkx8epXHMJBRO0dwly5SU6U0GSbb2854fjNI00ixWbZK3A4AM70a06/TsSjF/NsJyt0zE4iOpW2PveGpWFJve9p583n5tm9lKVtvsMLK/OcIdUEfPicULpbsrZ1bsfuE0gQwFXMdnTq5DxioScJA9HTV4u/Tty4ioKw8i89WMoGL6IZRWPPDJE3cBilQhRdGzqXOa1zP/J+HOfouRl0ESex3SFFTtHDRcN+IKT4UjF5P+176cJRYJUL1hYooHlXiWpxrG1fmZ0lXSsi1BE45mPxgjGVC4fz/P1iT9jlTN4eMjqQQuwUakRBeKEEHR9T8aupdTk2LAwLYj2SNEL15ZUDM31+8/djQ3lQnURphvQHIb+XghHodVbQygymNI7AvkVbmmp2KzJxK8IeKYrzUOPP8orf/CXb2nM5wm/SD34DuqcV5akHnx9FQ7tgJ4E1MYgHsaqKJAREBYoZBGOCV0HZASpHUgKIiEP+9RpSA4h2jqlylvQk2OnCDkh2qrc/qgGZz3JNatXIOjJxSM1Cd4LMFmAa4/iah2gREEsAhduGKwCWEsQbQe1BN61OyvM24RBfDvs6wkQC3rs1D1iWY/G0iKqppNSBVN1wWrZJKTIrEJbUCccUVBjMXKVJpVCjW4V+oM6U4rLYDRIr6HyRvGti48LGKleIiLC+fkJGuvOVUMGfJNIe+vqDcd3puhetyt1oFNXKQpZ0N0QEHahx1hzNBq2eIuN2bLbFFUh44nrV8FG7lQA+iY6RX78ByAVl8+EaUKjLmWhNb8QXdXh4A7ZeTtoQF9aBjGvZmCmJu3a5To8f1Z2FD9yAHq7pc3Q3QFjnbA0e9eH9TYIAdfGm+g66LqK4zRp3oL6/t3AO6JOrceNzsb6z93svRv3fSMVq/W5lgerIp2Hc8j41l6kF9wN/BUyQ/Fh4I+QNR2zwBkkpeoab3UmhpHdwGtISd0Xufl8nEJOlg/c5P3bYW75POWSg66rPHpkhJHu+wgGY2iGjmkvE9AVhFulfOIKpZUSwd1dFLtDVM/NkorBo0dHiQRGeeH10yhGkL0jz5COb0eIMmogyd5oD6OOTVDpQ3F70J0Yqogxc+0YhZxLdxKOHoFtYwpttkYiEcPR6xTdq1SWL2GLBu3GAAm3jYDrIgIqtlNHoCCqef78z8b54kX3jtJ4MdZS431AuCAnqHesYRNA3q1pSXfSgnLeV5Q1J0HVpHyj4kEgBOGY1G9XPAjF5LaGIu8jDUCRUrWaKh0QTcjFNNbycF1/v4akUVUrUmjEcqQyoS3Ai4PmyPqPkp8ai4TkZ4wBEGV52KGYVFghxlq3SNb9Xhd9eadOBryzoJeJPGbThWqmSTKtgJBKaK33XXF7+yUWg86RJOHeUQLpbXi2iufC6pUpHBQqIyMsNmDKvLMUsSfkz66OAJkFi5G93fyDv/lu3v2e+7n8/Gsc+/SnGN6zjUBbO/ve/SSXXr+2qfMfHktRyhTo3JVm59AI9sJpPKtGcXEea24Wkanx5mvLVIMhMs/ZeHi0vStMNCZI7R1mJAWJniqzV8tEUzF2P7mPldll4laVaG6cgCLYO9ZFveHy5nNzBLbp5I45CD96pQUh0RVA1YNYmsHuj/Rx6TOLnH1hhtH7OwgGg7hKkEi0G6shcLwAtXqd1fElrJklxvaNEYsmmX/xMqmHu+gfSlLJzFNaqFGacTBQqE6bNCyV7JJLPAYltUBKbcfAo96wMBKpTY3d3cLCY8K+QsfUFZbK+ziyvQNdayPd89t84dpPUrIaHNIfJOweoCiWWeQ8LkHSHKJEHpsl0ujEqNDgCg2qNEWOSKDOw4cf5oneA4wWwhz7zn9k2ctSBGoIwvEOhh96kPMnX2DSHefA2CCdBHCsR3k1/z7273+YvQeTuHjMX85y9uwkFxdWmbVrCI4A00gzWiFAhKiyA0+kWGEWjc3dd3cLz6dOtRDyZK+bnE9Y7+qTkq/1ArKW4055GB5YeViwpOHf1yezuypcpygkOmQU174NjVa6YfL5Dqsahw8fIvSnX6a6jiiuCdliwgy8g7kvYkDNgpFelINt6K6K4+qImj+vukBMQUQCoN8P0Y7rFm6ky8OY+zr5l38Bsft3QO0AxcWZuwrTx+XEn3g/jOxY6wQ3DJSb4MbkSQZ00H8C+k/B3Bfg9f8vzoM/D4oL4lu8XUqmCUZOWqach+qxTZ64jxuLYO8C//JgAivicvpCiWVDZypncdWzECjXO1gPJRTQAgwN9aBFw3hKkItTy5TyJXb3xBmfr1BxYTgWpqm8fWWIaCqjoSZfnl9iRgjCCkRVlbzroSGfJhe5NJY3OMbW6ugARU/gINgRV6laCrNNl6ItL4sOVLy1GttWfYcK7IkZnC/LbuEBf403gP0heHoT4zcy4DfjNaHZ9LN+/hoVNiQrr7QKKxa0dUPPbnmSfXlwzsPFqjyfSw2YmIFrRXjmMIwMyud6tA1em701pfh2MHQp+PPk0wFiiQR9Qz2sZhd4+TsFXnvtHfauuQvcterUjdkK5Savr/8c67a9cV8bbX/j/lqZEAcpYZtBOhK+LciS//tDyLqMfcjMxzeQMa8DyB4bIG+4NuBH/b9PIulaGjc3nJJIZ+VGT/tOEQg1OTTQT2/7foKBIEE9juPJzhS6PgC2x8wLL9CcWKb90QMk9h4mmCsSGV/GCFu8/MYs+fw0parLT378KOFwElfJ0OQiJecNXBEmqT0CIkEkksQTOov5KeZnJ7hvCAbTMDAMbkbQLDswJljNnCEQMXFxSIVGSDS7iJfrqIkAzVAKYS6CXeTChMVfHHOua1LfDC2KjYp8uAvADgVCCsT38Lbu1HeMhL9jQ+5cj0I4DkZIThaWkHUVKLJmIhiQKk9tUVAjfmG3KR84T5GRBc/zi7A0cCzQQ/J/RZUUNYSkwSqqpEoJITPzVlWenBuWTXqEALdD1n/EolJdorAMZhDUhPyMrcn7t+kAYem4OFXecrMp2jsYn5vgnbJ4HeQEuTrbQJnduPjudhhOayRshyvPvcnwIYhGmhixMKXFKfRoOxfeOMaZq/N3fKyGrrBrtI2f+qnDPHL0IJguV199g+d/5zRDD+7h8Y8+yuysy/zlKSIdafb/wCbSGUBPexdepUQi2YGuKyiqoKmaXFuYJ9OsMfhAnGuVMNmMLQs+OxVKExVCj3UT0uNoOjz6xBCx+CzV1Trhdp29T/TiKSUuvjxDedZD7V3F2NNBcjiG5wnsmidnY0dSvaculjHVKaqVKiFNkY52OkTdDjAw1EckGgZPxSZISHXIXDrH4kKdyGAbg0N9FOeXSe1M4tVXufrSIvlChmR3mPBOnUqjRsdYgFqxjqcrhNsThEMuZrZGZPsg0XQMs1693TBtiICmXOd83wk8PF6zXuR0tsYO7deo1jyiMY1QfIDu6H1Y1lfRCHFZvIGimhxIPcZYbBduQeFq5QoVXsTkdWwmqVNGwWYvKX428mE6Uw/wxsXj/N7MS2S8wvV1xQOeL06x9Ml/TNms8GAoyf1aP5X8EuOFBLHRhxkZjtPI1fjCl38PJTjI8OARjNIIsXwaWKBOlrAS4+Gex9gRfoBKHt4o/hkV/hLreuem7zKEjKa26tBcWwZEksjgioasPYsEwCwBFji3cQzW79ttwNKMpJP290M8LoP7qh/cCSdu7Wi0pu2a/7dj2vT1dXNfNMjrpnV9DtFUUEJQyL4DR6PkgNaA3f2ohkLIcKlqdTkgdSHVZLsFztIA9P5DSKcgBkpcoFhnKXzz5xC1WTj7H6H2s5A4Cdd+B3KzchEZ/QswdspO4BZSP3UwKCOeqgfYIHqh7VcgdA0mfxuujUHgDWhe4u0zqAtOQcp+NV+D8vwmTxx5oWPcVgTgZrBPZ7G6dYYGkgTSFuMlB12FrC3oDhvEA4KFqkNXXDA9s8wD2/tRU92ozSn2D/fQ1Rfj9ZnzsnlyqU5AD1+3p1qS+XXX4/RijnHkSKTCOo/t6uXzZ+ZY9O4u6WZ7AgeYqHoE1lmQHRqMJkPMlU0WHJn7bKmbCaBRtq5v3RVQOLqzjSGnQGrR45H+u+ermBZgSLaCaUq523wBag3ZCfzyCthLUsTkfZ2yPtbQZAPgdzkwc1KqSynI7OTlEjjH4RkPetIQDkqWhPMOnAFDhwPbwcAjkRZ4boGQUWZoCN58839CR+NmDsHN3t9omxtxI11q/Wc2yoqAdAqmgdeQmYwCfoQVWQg+gmyu9xLwKvAwkjrlIou/S8js5ylkpkJFZkNuNHZC/n5bBekm0mnZDGLeT+KUHQqmix7WCAYbhIJxouEUQRFn4cIxGisV+j7wNG6qk2NnLnLi7CQzyxb1GliWjRqA7dtC3L93N8I4R86ew3VcdLETwxkgaHSiCJ1qYZpS4TjNlbM8sqcO+2Uabu5NmMtAzgJvtkKfatK3cwjdUmkP7KRNGUA1lygHG1S9Es3mCtlCit/6ZJnyLdK1rQc54o/7MLIrrNEuvXoNiLwPgpvkTQW2S+UmxZOyjHoI2rqkw6B4fjMqX/LBEdJgD0XlAquHuJ6iF35kBkXWadiWpEbZii8jrMssRchPfTYsSaHykA5Dw4KyIxd23ZHa2KrmTwIy4I+hSyeotiwXdmGDFpZ1io5fDeY5vG3NuW2k0bj7ccuZt9/mTtBy+DeDYgVqNZOv/sGnaNp/yRM/8jBDB8dQwwpauM6z33yWicXbVK0CyXiA9z2+jR/72KMc2DtCbTnLsS+/SDaT44mP/wBuocqz3z5J1R3n6LuOsP++vVi1JqULp2Ta8i4xOT5Hf08XiVACRTgotkejXuT4czmunasRPhikrOTwCqacOCyBZyropkdAL1OzAmRWKsycXGD70T7aIiaxQ0nKqw0WsnXKHmQnmuzZC/se3Y9VbRCMT7L0UgVrFYhDbV4w7q2i2jaD+7pJDERo5CosL1YYPhTHjEWZmrqMaqQYGRzF0UI05upMzFfo3TFJZzrI0Q/uJZ8pU1opMHV1FdezaNYF8Q7oum+UzBuXiXf3MXpoB2a5SsKwQNSwmwbh4CZuOsC7hZPRmu832qLBSa65P88R5e9TKj/O5QWLbONDFElzWcnx4LadPNT1KB2JKLnFOpMr50nxV9T4BnWyxFDZgcZ2EiSx+UL2jzj35d9jDgcLOQ+1vtcDcni81siiAR1OleenjzPkaDzYP8gXlr/NpZm9xBM6faOHcJRRJuZjZIsZBIs0OUV/ZIiP7PoYewaHmbi8ymvV32Ka38Gjedvs/z1Fa/5RpDR6sSxr9eIJSeUIKBorWZdAWNYhBIRKMePdsunedThSdj1XA7MK/cOy34YrwDahvRfKt2DYCSQTQCDjRe8ZDNJslFmp1t5iWNpI213RpRLjpqCsooz0ExwMIXQPvbGA0Buo2nbZxM836LREGHdoN0RATQj0cJ76V/8FYn7GP5jPw+WvyZqL69EfTXYoFT7PpoHP4QnITuMmUmqoDoS2Q89vgPv/QVz4p7JT4ob5Wg+8KVAXwfo61Dfw2Fq8n/WSRhtBQ1r0d+pE3oBHD8DXX3LIl8qou9PsiJrsaDhcC0D7QBe25jB/dZWZisP+wV6mM0X600kCwma4J4mZq1D1pO9laRCyTZnFV2B7QKNkuhSR7JLWSJRNj77BAXbmGlycvXPLKqXI0835GXbLHxgLUFWFeDpIqGbKgA2yDH8AQJWS7ALpkz0aVdgZaBBueuw7EOb+9z981+OWz4IXkM348quwsApLGdmst+nCFUveKgEH9hRk77FQSNoNPR2QCEqnREUGRx0HrlbAOQlP7ZaBy7uI2WwI05K0xMKCQ/9YGNfVqVRganrzCm+bwV3XaNzOubjZtrfbRmzw941ZEwVJx9mFvICXkc/2TqQqVBj4a/7fBvJG6/W3w/97CukwXPU/txM5TxisFTHFkBHbFoXkIPAymze4hrtHSCRVohGdUCiMpiiIpoKnBShlrlJanKLj4adIdw2Tr9a4PPE55jNVWescl30OVrJQqUG1Mk5bT5i4tgfX60d4UVwalFeOszJ/klxhhmjAYqAGoyMqlYDHsa/C3BwUXFi1YOHNGg/UAhSyOu2WRcfROKJzF0WjRN5dxiyNszQX4rf/YIZz4ze3WFslEmFglwbvMuCvmr7TMSr10Z0IiKQM3GwGqTSkY1IOrhmX2YtYSHbdtB2p8GALCAbBrssGVEFDar27yIyEja+SIvwCbn+9aGVhLFsu1J7n989ARhjqTZ/n7H8uFJRSsAjplDiW7N5tGIDmKxUGQI2DWwYqcqF2IviSE/5Eu5F3fTP4xeV3i3vP5L17JBIGbihOencHM1dzfPXPXsD42psIp8Zj7+rg8uk5KrcpztgzmuJXfvHjPHBgiMlXzvLsH3yZSKBOPFSm5/4BXn3+NCIU5tDT99EeD1PJlrj0ndcpFnJ0jPVs6rhf+OQcBx5Pcd/uIomYiqcJzhxbplB2sWqgRqMI24ZlW3LXYwq9j3URClmszs2iNTQSj+6kVoOps0X27vFQGxZOxaIhPMKHk+zvj5GfWqa7XcFLJVj1bHqORqjMNdGEoF5VcQM2YUVjaCxF6GAHtfkpIjgsXsoTywkaVoPuHUdxh20aUyasQEO1OXd8Cq0epO/dY6SCdXqGorR37ObCxWmWxutctlbY90QvRljD8DJMPlcgN22T3K7R2RPk0P2duO7mVqGkkqIgbAI0MPGu39pxFUZDMF7fWFRDQWArl9i77xgvXx7hSu0SmhPmwfjHeOToGEpNobxUY+rSJKXK11jwvkKGRSrUCCDoxCWBxwQ283hkeXtNXguCtT5KCvCsVecl6yqPa1EeXmjylcqfs/hNBRSDuNbNw7F/RznfQ0lZJmkIRtMP8ol3v5+oEuGV187w/MKvsyC+gYfFgB7mmdEPbmrsNg3fe/M8yC1D7zC4ASnRrekyIFcsSnEGT/fWAh93cok9+VNdhGlTdjnu6JC1aOkuWE3LdhQbYX2A0ARKvbuYyLpM2eItU55QoFGFmAb6Jo1l2tLoezoICghpBXT1/6JY+DG8Lg9cDUUDVXjYl05Ceg8EY4iQjXf1k4jTL96wM/Ptc3LUL74zFNnN1c9Q06dIo6HsgbkCy5MQeQiG/iuU/z6svnLzY3ZPAfNSWWqjabAlf94SELnZpG77P6HbjNFNkJ2UxdhvLLr82I89xO62Ml/49Eu4wJlrS/T2xFFVBc8RLJVqZIpFxFwe4cHEC2eur6c1D9yaSR2ZVfvZNoOHtyf5zMks885bD7/menznpZPs6Y+xElTJmXcWWlc02JvWOJtzKa37iAes2oLnJ0vAGpOsBowDMT9h3IGkZp0seqjn6+yLwGLd5tjXLvChf3V347ZchGIeZhdhoQDzDVh25bGsjyfaQLYsjz0YklTBWAzaw7LY20U2aW7hWgU+d0ZmM+zNGp0+XE8G/KwaFAsB9KDNi895nD93ZyVB9wp3TZ2CjZ2DG7e91XY3vu6x8b7Z4DMasAM5cSlIJyIF/DRS4eJ1ZGZjxn9/BelY7GZNPaq1/znkDeHXuqEhKVI2XGfZWkiqVidvl7+9U/QNhdDVEKqi4HoNFs+epnHuOcx8jPanH2PvI/+Atp5HMe05rs59mprdpDMpi/zKdXlwbW0KAd1mbmWKsZF3oTsWnncc4eWw6guUy0vkcnUUG1QbZlahb1Rw/gSMZyBjy47U6TB0RuDkFYvgpav8+AeHUTt3kVGmyDRP0CgsMnUhwO98Kse15Zt7B60sRhjoV+HdnXBhVdbFpAKQGgB7GdpiYF8DOwf87bsfuzZf/QkHomEpD6eHZPZAVSXNxPCkoyF8QQ6hyAJwT+F6h1tbyBSm5XOVTROqmhQUsS1fZcSnTAlNPoS1hjxHIyAX7qAG2Qrk6zKLIlxwa3Ifpi2dFOGCCCBvqibQAJHnpouEZii4t5hNFFVB3EqO6Q5ws2fqu41LCyaTl5d48skjPHDE5rf+zdeYOF2ld0jhkOXw+OO9LNaznJm3bnpstXKd088e58prZ9jZAQ8/NMLJc8v87mevMrwvwCd++gcZ7O7i2ptvMHXiGq9emmbHkR38wN/6afq3HdjUcR/84U6621O4eomZy8sQ8lhddSk0YOy9naQOdjA9laXWZ2AMA0EYHEwTcCqsnC8RiobpHOxk6GGTcrZCJN6FmuxF9FRZrZ0iOpSgO6Ux8+wi52dniHSrzF7x6BzT2fVogmQyghfowW3UUKw6xWaTqwt19FKN0Ipg/hQoepEP/5N9dA4FmDz2PA2aUvlACJaeq4JaJY/JwXf10qaqxKMaKVVFaUKjUGF1QkhFIbtJdsIlsyjIeSpzMxaDOxRWLlzY1PNqC40ge9DI4pIjHNRpODWqbp2zdYEM6QTQMIijEkQQDhocVKpUHY8XT41ycPdO0kY3X7v6FUrWCWZP1qg2l5g2X2VFvMBeFlHxWPUdmQYyU31uE3e4YK1J7LfcGs9VpmR224OgYvBQ+z/HLnfS1tbgqbEDBPQYPaNBqsUarxz/GucLv4/KOXQsQsAvj76bZx76qbsfuHuEagmKq5LWhALBiIvtyowCRTkNOQHJ9hGmDLrg3gGFwgMzC/O+rG5nJ0RTMLIXLr66sdGy/iULeP38OINjY4R1lYr91gmxkpdKdxvQ++8MI714uobleESUF6k1v42Y2AvRPnDbQVGI16ZxFv4jbvu/AWMHzJ3H/c6v34FcrADRgNKSXDyi/TLC6Sh+YZ+ApAr2OCz9A1j4Ddj2OIz9byD+V8hc3Hi33jlJybA2GHz5mFyXMX4LR32Dw6OOf5HvHlem5McDChgTEwSbMbmuA3Xb4+JCGVMIXAErxZK81kLW0icDGhVTiiyMaHDfrk7KdUFmJst03uZaMcsxd+Plb6pg0mXZbAsot83At07d9GT7kk6kE7HeV/Z9YpIKNPxEkx/7ozOs8wN7+7nf8/iPp+Yoe3DYgkEBkwWH6sIqH7rLcfv6q7Kp/IIlM3cqG/vuHrKcx3Fk7YViyNrR6C0cw9w9lLyfXoTOBJw9ucLwDo2OLkEsKimR3yvckz4asMZ9bb1+q1qNG7G+DmejZ2n9a626DAvZCOh+ZJF2F9LB+DLSWVhC3mRl/z0Vmc0AaRyDvBEnkcakhbQL25EN/EB6vzmkM7ObzTsajudRql2mXi0y/sI5kpkrhC94ZOqC1Ed7WTLPsDD1KpaVR1MW+cH39GJ7JRRXoKkaquKgqxEUggjXYWnhBJYZoz0doFRYopTJkpmSeTotCotBSQGauSxYWYRzi5J+NNwmjeWZqrzZP3YAOgY6Wa2dpVI6jbba4M1jYX7viyWK1bc/Mq0aDM0fsyjQq8B7RyDQUDjhCalKYkAlC1peKpYo5c2riQRDss7PRtKbMCGiyto7z/MnF03W1KlBKPmKDzGft+y5ft2ILdcI4cqajbIpU/UBXR5boy6jf44j05jCAqcBSlRyJS1PamKbDagUAB08CznDKHIfbhO5QGistb+Ns64z1rqxlDbhhlQTXVmTen2nTkabDoMRmKlA8XvsaTiuYHp8lcLuJg+89xBPfGSC3CevsHPQIDdbZOdDP8C/vE/nl3/t81xc3njWW8hZlIpNDu3poVGo8R9+9WucWajx8R8c5KkP3odmufyP//BJ8iurjB7o5+/9h5+np3uE2WOv8+xX/pCP/MpH7/q4K7kcSr6CGbe5es2jmAO7DcKDMDKm0NcdoivYRX5QxVQ9yiVB1GiyY1s3hu2hJ9K45Rxj23uYFxb1UoFgPEWqo59dD7nkLh1DMwKomhQQyJU8NKHTu7+P2Tfm6d2rsfeRIQyzzPlXTrBohVl+I4NmCnQXzCZ0PWCQ6knRXM1z8eQKoeF2UtsMis8tQllAt4JVbJAdz5PqNaAnRrrPo71LIbcoyKyYJDpVUokoiU6bTMXEc1WcaphrCzWuHStt6ppXyAEXiHCU98Z/jn/yr95HM5Dnv/z6b3J8+hpdylN4IkyVVRqM8+7gKL/w8z/F9OXX+M9f/AMuL/0pV1fP03CjNLnIpJlnzrQwyeP4khKXkHPzXkVlESgJ757ozQveSqHtMrZhNZMc3JFAFxbL+SI7hwJkTs3S7hb56zsH+UTiV7l0+TyfnPw3JFnmid5RJl/5NP187B4c0d3DtWWdmBGWgRUhZMDFQcrHpoOS9lkUkvah63Kec2x//rrVHCFk0fbcFTmvDg5Ddy/MpaB8k6zGuo9ycnwO/c//goZzw2Lgct0S3ewUpbSFcZtN4t2nsfhtKpkJmPwtiHbBwNNgNolYHgVlCGqT8PU/RCx8Bup3IuvjoVSvIpRr0DwK+/ug7kev6r4XEPAgUANtAZb+Bej/HbY/CO3/EKz/DKXJt+/W9TbW1teQ2YlWaBzuUC3jDrbZAAJJdx4LwanXJ1jJC1L+e1FA8cRagNb3KDVk1mKXcNkzCjMZ2OEp/K0daeLDA/yzP32Fv8qZOO7ND6sGvFbzbptYW+9jBT2YWHWZFNKBiCugaBqaohBQFXpDYQ6PbENxLBazSxxfLWAL+Jl0kKNBj69M5ski63TnBJy3ZC+14U2M22RJXr7WJbzV8JdMqNchmZS2guNK/7ZlT93bzihvxWIWCgVYzNbI5hQcx69r/R5iU8Xg67E+E6HyVue7Ve2/frv1DoNYt/2N+xQ3bLN+u1eRRdzbgaeRGQ0LuQD1IJv1vYosAj/hf2baPxYV6Uz8dWSB+LP+Z5PITKWBvOgBZM1BDemAJJDOxmZw8uwXScc6mHqjSP7VS+hVKNPJoX/5E8Q7k9TrSY6dPsbXv/UqS9NVLNPFEx592xUCUQXFFXimRakKyxUH1xF8+F1RnnogSE8iT9ASTAqY82AoBUf3qTTLEf7iU3Wee8WjrRN2dUGzDrO2jP5/aB8Mdiq8evY0vXsaeDMD/NVnr/LseB5nA6k3Demgqf7vGLI/0mPb4f52gz86blMA7tsGP/M0vP5l0D1pZ+cD0LNJVYxgQBrlnicb2riWTEF6cD2yIhwpNashHQnLlHQn3a+dsIQ0zuotWpTPaTZdGZXxPFnQFRCS4mV5YNekDG41LKlRloBsVf7G/x5sZCjVRj5JGmstTBv+b5/DpxjSeWlhfQSw5XS0EA9AQxg0bXvTK2/rWdk+GCDmuVwsb9LTeydwwc5aHH/5BMmuJLqhEIkr2CWPa9kGuewpfvqf/x3+6T+y+KVf+xJLGzi3LvCFV6Z49dVJxqsOCHigW0PJl/mN/+s72I7Lg48M8jf/wUcYHN7N2fMzvPDpT9HIrLLn4cObOuzlOZhrNAjoLqUiYMLwWAgl36R6bhU75IERwm3UsYp1issQb9jkAwY7do+wOr/C1IlJkjuHWb6Y53NvvMjowQTD+wcozhfIrxQo6CHsONSroFVg8Gkdu1Bl5ZJHw8zT3Xkax21y/mSZer4GTY/YoEFHn6CWddHSCqdfmSOWDFI0QsQ6O3HmV3yPHAKGTrxdpZ4vUkuFCbsR9HCUkccCeK/V6d6Vwi0WufJCmVS/SrBT0FQ8vEaNE795HLe5uftFztM1arzCsVqV3/ijOWLJEOOZccJ6io/v+RnK+QCFWpPl6jRdXaBHdqDgkeBFVniVa+5fYQFBPGygE50KHiZrbJJ3d+3kEx/7KVTV41Nf+wx/NH3hntIFFaChmLzv6BAzizO8fPl52rQAq9N1Hg71sRLo4NSqh2VoqHaQYfUZPO9PCOpBRuNt9/BI7h7FHATSUulO9+cP14GGL3ARDEnaRmFFzmPBqDTgnCg0qz7N9RaD6VRg/ooM3AwMSEWd2zkaABnL4zsTuXekonMziIZFMHoax/k/MBtv4NaqEDkOC2eh972IcINy8VXc6p/Aqd9kjTx3J/AQS59D6/8FvFwGkZsBbQgsRWqml/EtxrpME4kTMP+3IPBrENsHgSdQ4g1EZfHtu94o3dyS2bzx9ds4gXd1SutwFRkvC1jg5FxoyuWrFUe7cbchBQ6H4ImwwvsfHaXngRSLSyanvz7JN794lS5jnGlbbNgy4Ea0ek/eCi3ByVa7AVUoPJ7s5NDAEMO9nSQ72uls7yEWj1DNzNKTCNMdNMjPHuPf/uXrPNcUnFusUVis8Zr/nWXgt/2/DTZX3lJGZlYc5Hip/rF6N/yAtDWapnQuPEdSEB3n3jXkuxWEgLoD9RKsnvhe8xok7srRuJESxQ2v3ZjJWO8wbLS/jf5ev//WRVB56z4eAR5CGrGtWikT6WBkgO8gnYJvIgPLrePwkE5JlLWgQRuy7iOIVLrOIKNlGlK5qoR0Uh5B1mlsBmMD76UnvY9v/Y//xOKLEP5wiAM/8Qi1qMlrL5zkL791lpeOTVNbduQB++5t2zx0DEtxi48OQ3gXXKjD4cOwrbvKSrHKK8dkNGF4BD70qEo0GOL1NxX+6rNNSmWP+x+VuvznZiCchkgUDvfCfb0abyx6lEIGudfgL//qFPPFjWUrdKRjYawfMxUO71V5+L4gcy87TLtcL4ge3i+N9PkTclHKV6F9aHNjVzVlV1oVWVzuOPLBsRzpPOgKeKZ0PlykkyCErGuJJ6Vxb7lQbzkVvvRjsyk5kCVL1lrYjl/P0YSoCc0SEABRlZHBsiU/E4vKbINTBbuEnI093t6IpZVHbTWCXffWeodddsTVwPPQFMldLpjwTmMc0aCC7gjcqs3VsviuRkxuBqGAEoOVuTyzl05Td4r8yN8cY+aNOdpSXSR2tPPad57l0BPv5ud+dIlf/cPXqW2w6ixWLLywQk86wJ5OFdP0+PzJCu9+cif/6H95H72dKRYXMlw5fowrFyfZd99Bxg5up310+6aOOx2Pk7dLFKahc1+MnUd7iHhNqqUC0bJFKN2Og022rFGc8yheFSTDFo16k0gsQiQSAg1sK4fiOUSSOkZc59wrFzGSAcKpGLllyF8TKFHY82SEjrjHq1/J4xagOCM4d2aRZFeEZlaBtiAsOUQHDDr6BYOjEeI9Q3z7f1zCrLkEdidJjhl4FfM6qdcq2xROwcgHdQoVi+rlOm29UeLtMZLtc8ydLZDepqPGHIgo3PfwIMvLNapFi3ymKh/4TcFAasLEKHgn+MqFN66/oxLhq7O/zp74J0hHhrHMfno7DS6eWOE3vvyHTIrjVMjh+DO/CURQOcR2ZshygRwm8IDRyT/76V9j9J98CEU49O8c4+V/+Q+56hcL3IxhcqdQkXNe0Zzgvzz7Y3Ri8ovpQ7wi4lwqCv6iWmSBRUrMECbMqLKfbiNN1YqTX50ibX8fnPp1ME2oFmXHcAyIaHJZCfrU0oYna9lcVVI4gooMdpTKskGfo0l1qltJZLt1mL8stxkYUliZEzRuo3rk8XZhpPU29TtC5QKu/UsUzZfxyq7sxtYv4NoVqDTAalJf+iK4m5R0qbyBN30BeodQlq6i7BjG8xQ5gSeFLNqLNbiu7OFdgsmfg8F/Buoj0J1Cj/4xznJmbZ+t4FSL0C9Ya/pgrnvtxujsRrDYtA7qLH6NqisZYrNImvkMsrHxjWvXfgEHLNjznp3s/ZGfRIvU6Xx6B2bks/zqf/sqFyxB6TaHi/+dvf73r0crYK0i6fBl1u6bbk3ngw8+zTNPvB8FG83M0Bu1sSdmmDg2RWZ2gslmjabjMdN0WfXH5C+Rtl+NNaplCxayrvduUUPaQwG43nXcZyLi+a+3pJ0rNjQb0n5xbVmkbfqL8vdI+On7iruiTq3PVrRw4410s8xH673176/fV+v/Ww16a5vkuteawNeRqb8eZE1GA3gX0tv8DvIk51lTSn0AeRObwCjw88ju4TuQWY5t/vsPIW/AAFIFe7OL16WZl5hdPc5DP56g8UP3YQVneXXqOWZfNJlcsSkInZ37NMZ1h+oyMozgSCWDagn2xEFLymwBMwrT47AwbXB6wkKLwpNPaezs1hEVnWtXbL71TYudh2DvCLz8Ahw7B3oK3rXb4MhwFw8bWRaagmUjyvnjOq+8epGmtfHZ6UiHLuKPQ5sCe5Lw+AM6O3fHCZQCPJ9fpeiP2fIyTC/BzvdCTzuc/Q40VKhucu01DOk4KMKvm/CdCxMolyX9KRqWRYQmIFRZc+H59S26sfZw1x0Z2RMKuFWZwlRUGdWrVyW1IKj63cLDoKflGtVwZF2IpkG+JhdhXYPrmssbDd1NbuT12QtNkZkX4fPKoiGNeCTEQq62VjS0ycWjagpU4GLmziJL3w0EQwr9YxDI6wwOR1ES7ew58giXQq9w5swS6WCKV798gtUlk86eNB1RjVpp4xvF1Q2isRCvTFd4fCzFf/uF9/LA+5+iurLCmROvUahUiMV6eezd76OQnaG5tERDDRJO3f1xhwajePN5YhGVvYc7SXZ2MXdlltljDQ4c6CGU6sH1moQyVUr+4lHWXeYWK7j6ArMX5zCNAM5Sgb1PDtLe1Ul+aYVLL5kMPBoiN1fDSMfRwgqaEIRVG9XzsFcBT9oqiyctloM2niWkCkQZihN1FAv6+w3SwTC9R9qYOp8h2mawc/cYolxg+tWKnOhqQEIjEO8iO7dMfmmVnWaSg0/uwOoqMTtbJRYLE+q2cRzo6orS0x5CC0R5fuICucnNPbAaKVzKyFk46L8qb2KPOpdKv8fV0vMk+UFsBli9EOMLl1/kgvdNXLLcGJYyETzLJB4KbSR5LHqEn/vw32X4Y0+jthkoRoDhp97Hgf8+ysRkgcPITPcbyADR7bDRWrUWN3DIMS17NandvKuzky8VX6GETZ1VBCY1oCSm2ekdIKyGuVCYojfatykqxr2C60KjDI0kuCFZ96AgM79N37hxTNnrp2mDMOQcGvbk/JevyhoO1QBMmUXeqAbDs2BxAoIBwegelUsnvLsqMFWQ68rmhJRvwOm/h3f/SbywB00hec7LHtS/DBMfhJ37sVYuvYMvcBAz/wGmqxD/OGLHu+WClzsHK+Mw9kMyKrWeFibmYf7fQtcHoGDjxj+AumMOb/kVaGVvPfymgFyXcH8LZQrWPN/bRYs2abGeRsYVBNKon0U6phtdyhRwH9AjYGB4CDs7weK3vwqdbbLWSYH7egzIO8xYgulbHFaQt9py618PA6ORCK7rct40CSKfycf60vzwR55k6fi3ufbSyximhQmcqjS54IrrCmY2ckhtpF0QAoq8s54UN0LwViVSj7cKRK4vCK95UKxKtoXZ8DuIf78W5u8D7kp1aiO60/rsw3q52o0ygq3X11Oq1m9zY/bjZnUe67+jjrwp70eurYvImoyryIzEk8AhpLTaVeRD9AAye7EbWYPR7b/2Df/vOvJhavjbdyPrNDY7Td2//cOkYzu4NPUm5cYZQupuDo+lSLV9mc5CnfxSlKnpHEsp2WjU8+XslDB0NyBiQ0cXzH0HjlkCw4Bop8VDR2BHm07VDvKNbzZ57y6TQ53wm/8A3igH+R9/YnJtFXa9T+PBw3GCVpQRJ48XCfL8jMe3vlpnbta56eKgICegGDJD3JGEB4bhiT0anYMpInqM+nyWFU9BQUiHrgyf+wI8cRT27YD9H5Vpwokvbm7sQrpUS9F1mS1RhJRxrAvpRIRDskeF6fjysgGpItVsyPvC9GsmIgbUfB1kT8i1yKpLR8bzfAfEkZkRR4OQIf9uuvJ6RHSZ9jSLyF4aDeQNd5fep+Gn4GwBNzLUTFehO5WUepLwjmdFj00rHt4TtEUUOjsMnvyR9zP20AMsLiyiBdpI9PXw+idP8jtfnWOh4aGdfIWQoVI2b27c5qoWB3el+d//2aM8+sAedCNKY+4ac5MTdGzfSXswgVF1iIVjBL0ullcs/vxz3+SXfu+f3/VxNyoreI7AsxSWV+u4oSZX3lih2RSEe7tIdKUgn6MwV6dmKChdglrZ5eKrJc43iyQeC5Ea7mDhSp25s7OkUouEezRsU2DXbZRwgMJ0gfZeGUgoVGyoyVsp0AfxfQrlWYEtfGpGVUqmhDuge0hHGCbTi3MEUx7hTkhETczlS5jVLMoQiBAoWYXeR9vRoiqFyyrxUYPe4TCN7Cwz51dwhUvmZAEjKQg4ChOnJhneOUjCLhEbDJCb2Nyd43IQlV1IF2Ee0YqakAd0FHoQRCjwVYL0Me0k6PQGOaD+rzhemQpnKDGBg4uKRpokPfQxQA+Pth/kYx/6AD3v70bVVBACt+awfPYyC4VVNKSE+ScI8ig6v0udi4gNDR1NgX3bIBWC45Mbq4u2MAe8VLzMxxKdpIM5Fs231q/Y1Jh0zjCopPiNxUvcF1jifZsavXsHz5XZWasp5zfhyayt41PrFFUGXxxPZoxLlmzAZ3pSgrNek0GRYEwyAPKFt+v5a0kZcJm+DINjglBUqkfdKQT3yMkAFP04qaEI5WoDJyLWurWJLCz8EgT+BhQ3oC7dDbyc/F3+Jsy8Dn2HQf0SzP86qFm0oWVc94ZB8rKw+nWU4fvwmlWEOQztA1CbfqtsbYsf1KJevO27ueu15k5x/g63U5FBWBeoCFh46TWufdEktGqTERmWgG0CRtsDXGu4vGlt/Oy10FL5bKHVUiCkKPzE4Qd45ukfJT83zS//xe8iPCi6DtOZPJef/SNqxSqzlSpX6y6r/jGtcvP1Luhv4xMa3pIo2ixSKhT9E2zFHBWgR5FOTX3dF7hC9tlo1KWzUSyv+Zr//4C7LgZf72zc+DqsORA3OiQ37qvlcNyMjnXjfm/cd+v1IPIZzfh/dyIzHK8hG/f9INLZCCGLwQeR6bhVpCMhkAHAVaTTUUXWcPQhO5A7SKrQ+vTd3eL86Qp//Bf/ltXqOH/np58iHotwfPo7LGdXsGohAobKu44+ykBvjj//0ikqwjc6ItBRgj2HwZ2Gc2UIPyIN6sQC9M9C9x4HT3eImZBQpMRrQIc3L3p0HFA40pumO9XBymQdvbDE0JEEJxpdnLwyjR249Z0eQjptYQVSCdizHR5Nw1g0iBfvRi1XqAaiFO3a9WtjC3jpPEyPww8+CR/826DW4Pcn4Jc3MXb5OtgN2XQqoAO6fFCjMXANmbFeWgE9CJGYdEiEgGwJekJyIdQMWZehVKUz4Xp+Z2tPHq9i+gtxAzw/AmgqEGrIAFXTQfbZ8JAhkjpyJtkEJ9ZypNMi3LdL15mWw/z8CqqhENQFzebGkcT/O6AtovLBhxLEYnD22CTHXloilvYYHZknMzdNteoyU5eqQbYQNG/hZABEdYV/8mMPcHCsl2qljKaaeM0aw8ODmCLBzJUCL371FU7PZri2UmcqU6VsOvzS7939sS+eNDFnQNQ95l/J0liqYE05bHuqDTWlUazaFOcXqZkWoU6VhudRX3BleMsFe8rEDi9jZV3CaVi+ZtORVNHCCk7OpKcnhFUEvakQj2h0DsbILdUI7rExOsH2BQaUDgMR1GFJ6iI3p0Hdq+PlLfR4Hc31SMU1VFFneWaSVESQEwrxToPIqEdUK2Ovajh1l/7RNiKpOOMnMixPWySP6JRLDrqu4GQEy7ZJe2eFsuOidAiS94VvPUg3QZ/yj+gJDNKwVpgTv4zJTjQC2NQJMEJa2U5/h8v9vd0MJ8YIG2k8J8iVixVmyucoO70MijDdeprOsELc8xgIpzh0sI/9R3uIJAyap5vUTl0hrkQ498Kz/LP/8uu8mZ/HA74F9GLzpBblF40Yv2vneMF13mbwdA/Cx38uRsiwMT9p8uYb3LSfjQm84Db5oVKGTyS3869XT76thKFJg2nRYAKYNrP81qZG751DQToIeH4fIF/gQnEBVcpq1qvSIfD86b/pn3cD0AIQiEpREc+WkVe9Dbp6IZ+T9NEW3CqoSTmPzlwSaBst9t8jhPbqRAMWFcuTc/QQcqHPA/lJWP3X0vsKIQ0FmzUD4K6Rhcv/Gfp+V/JslRrM/2vcenrjzb0c3tQLcjFyTkC8XVpe6yParTTaRhny9WT/7zNiSDUqR8Dxi1UcpG0kkPZRVxD++GKNFfF2NvGNyCNtrhYcZEbggz19fOCH/w7F0Ai923bz3vMvc2ViCbtZ5Xizxvyz54grCniCeWQxdhjYqcBVseZsrM9W5td9z71wMgDua4cXMm/dn4sMIPYaEI/C6eKa71hrSGejWYXxJaj/T3JNvxe46z4aN2I9Bepmzgc3ef1G+tSNtRk37mM95ar1O4iMYrXqCP4T8gZ/BvgosoFfB9Jx2IakAZlI5+F+/zO/hWzgl0PepEeRqlWT/n7DyAcgdcuRuDl+8uf/PabjMTgML114jnQXdCaHeWjb3yYa7kegIwiRWf0SsYhCwxR0tEFYB2UBctNQ7YXCYQiEYeFF2OlCz1+HigZf+JacU7MdsGs3LORC7H8sQG/B4cJrdS7mZukINPl7jxhMW1186fVZPN3Eu0lKvIV2f5wcIQuUh7KwI6UQU6BGHTWooEZipEMFDNO6Xgc9oEhVp2NnYOwaHBiBSGRzY9c0ZRagYfpZaUfSpfSgjA5UGn6UQpdN9cKa5D/WTZjPwWA3qI5Ph1Jl0yrRAKUJXkNmja5nFprSoXCRi6figRoAryqdAnt9OKSVo70NVFUaMOuH2dV0nJv0KbA88GwhnZt3MBuG/GzM9wvtEZ3+9gAr11ZpFoukk0kGR3cSTxrMX/XINLy7muyrtuAf/7vvkA4p/PgP9nL0yDbMYp2JqRyLE0X+8MUss3X3nhQEK56s7TES0PdIlM6OdraN2GhJlYnj1yj1VslPZhg5mCQwlWN8zr83fTGA6qQg2GMyuF+jo0fhqueQOWtizYHSZhBCEBAewa4g2/e1o7seV94oYSShOAkUBaE2neBQL9WlhjRkkPf7hdebdHYq9LbVadYEruGychKC/dLpePDD+2nrC2KbVRpNjfEz04g2QaVqMbNY5OpLBVxToa27DeFl8IpS/r9rdxhTh7qrEIgFGHt6dFNjN2qM0ZNOMZ1bwrInsDlBG0PE+RvA/YyEu/mpJ7aT7oyQrwjKBYd81mKoJ86ObY9RrjRZXiiTjikMt+v0pXUe/tl20sNBlAse9jjMvrzClaVvo158nT8//gJvZqrX14w88BIewzR5KNLBzzRc6o0cb/LW3g0kQUtEUQNN+gYstBMCx5JrSjsqQWDRl88VSHrJS6UJnu59hN/VQ8xt0GK7xWzZZBXAvYFfb9ESclJ12fbBdWTvDLfWmmPYsBGc24BGg7cs0qVlqAUhHvHn2aYM2OBIpzjYBlZF1q19vxDs8WhWPEQNGVFcRUoJ7UIu3sKVlrDl/18CFpCc6s1QWBrHUI1reFrDtySLkCne4gPOWoe0ysrb3/5+FNHdJVTkLTONtIV0pDrvov/Tq8kg6Ip58yFt2W4GsLtdY7bkUvKHxUHaZ3v3HCYaijP37J9it3n8Lz/1fv7tf/l95ms1BDDvwY03rgvsCulEbJfXHXGdebzBLX7PEkP7euB45u1ZuTKwKyRrP1vwkGIM5QosrsDlgmzJcps2Uv+PgSLE/11jplvYwha2sIUtbGELW9jCFv5nxSZFR7ewhS1sYQtb2MIWtrCFLWzh5thyNLawhS1sYQtb2MIWtrCFLdxzbDkaW9jCFrawhS1sYQtb2MIW7jm2HI0tbGELW9jCFrawhS1sYQv3HFuOxha2sIUtbGELW9jCFrawhXuOLUdjC1vYwha2sIUtbGELW9jCPceWo7GFLWxhC1vYwha2sIUtbOGeY8vR2MIWtrCFLWxhC1vYwha2cM+x5WhsYQtb2MIWtrCFLWxhC1u45/j/Ac8MJgmx5x30AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
-
\ No newline at end of file
+
++ +- [CODE] CNNImageClassify_TransferLearning.ipynb \ No newline at end of file diff --git a/Pytorch/3_NLP/BPE.py b/Pytorch/3_NLP/BPE.py new file mode 100644 index 0000000..a1c802f --- /dev/null +++ b/Pytorch/3_NLP/BPE.py @@ -0,0 +1,93 @@ +# collections.defaultdict: dictionary와 거의 비슷하지만 key값이 없을 경우 미리 지정해 놓은 초기값을 반환하는 dictionary +# re.escape: 문자열을 입력받으면 특수문자들을 이스케이프 처리해줌 +# 정규표현식 r'' 의미 : 파이썬 정규식에는 Raw string이라고 해서 컴파일 해야 하는 정규식이 Raw String(순수한 문자)임을 알려줌 +# [정규표현식] +# \d : 숫자를 찾음, \D : 숫자가 아닌 것을 찾음 +# \s : whitespace 문자인 것을 찾음, \S : whitespace 문자가 아닌 것을 찾음 +# \w : 문자 + 숫자인 것을 찾음, \W : 문자 + 숫자가 아닌 것을 찾음 + +# [정규표현식] - 메타문자 +# []: 문자 클래스 +# {m,n}: m회 이상 n회 이하 반복 +# | : or 조건식, ^ : 문자열의 시작, $ : 문자열의 끝, ? : 0회 이상 1회 이하(0 또는 1), \ : 이스케이프 또는 메타 문자를 일반 문자로 인식하게 함 + +# [정규표현식] - 전방탐색과 후방탐색 +# 원하는 문자를 검색하기 위해 정규식을 사용 +# 전방 탐색(lookahead) - 앞에서 찾기 +# 일치 영역을 발견해도 그 값을 반환하지 않는 패턴, 전방탐색 패턴의 구문은 '?='로 시작. 등호(=) 다음에 일치할 텍스트가 오는 하위 표현식 +# 후방 탐색(lookbehind) - 뒤에서 찾기 +# 후방탐색 연산은 ?<= +# http://minsone.github.io/regex/regexp-lookaround + +import re, collections +def get_stats(vocab): + pairs = collections.defaultdict(int) + for word, freq in vocab.items(): + symbols = word.split() + for i in range(len(symbols) - 1): + pairs[symbols[i], symbols[i+1]] += freq + + return pairs + +def merge_vocab(pair, v_in): + v_out = {} + bigram = re.escape(' '.join(pair)) + p = re.compile(r'(?': 5, +'l o w e r ': 2, +'n e w e s t ': 6, +'w i d e s t ': 3} + +num_merges = 10 + +for i in range(num_merges): # 4번 과정 + pairs = get_stats(vocab) # 2번 과정 + best = max(pairs, key=pairs.get) # 2번 과정 + vocab = merge_vocab(best, vocab) # 3번 과정 + + # print(f'Step {i + 1}') + # print(best) + # print(vocab) + # print('\\n') + + +# 한국어 +S1 = '나는 책상 위에 사과를 먹었다' +S2 = '알고 보니 그 사과는 Jason 것이었다' +S3 = '그래서 Jason에게 사과를 했다' + +token_counts = {} +index = 0 + +for sentence in [S1, S2, S3]: + tokens = sentence.split() + for token in tokens: + if token_counts.get(token) == None: + token_counts[token] = 1 + else: + token_counts[token] += 1 + +after_token_count = {} +for token, counts in token_counts.items(): + # " ".join(token) : 띄어쓰기로 분리한 token들을 음절로 만듬 + after_token_count[" ".join(token)] = counts + +print(after_token_count) + +num_merges = 10 +for i in range(num_merges): + pairs = get_stats(after_token_count) + best = max(pairs, key=pairs.get) + after_token_count = merge_vocab(best, after_token_count) + + print(f'Step {i + 1}') + print(best) + print(after_token_count) + print('\\n') diff --git a/Pytorch/3_NLP/BPE_Example.png b/Pytorch/3_NLP/BPE_Example.png new file mode 100644 index 0000000..6533be1 Binary files /dev/null and b/Pytorch/3_NLP/BPE_Example.png differ diff --git a/Pytorch/3_NLP/BertTokenizer.py b/Pytorch/3_NLP/BertTokenizer.py new file mode 100644 index 0000000..54b9dc7 --- /dev/null +++ b/Pytorch/3_NLP/BertTokenizer.py @@ -0,0 +1,17 @@ +# Bert Tokenizer + +from transformers import BertTokenizer +tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # - (1) + +print(len(tokenizer.vocab)) + +# (1) : bert-base-uncased 라는 이름의 이미 학습된 모델을 사용, 해당 모델을 사용하려면 모델 학습을 위해 사용했던 Tokenizer도 일치시켜야 함 + +sentence = "My dog is cute. He likes playing" +print(tokenizer.tokenize(sentence)) + +tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased') # - (2) +print(len(tokenizer.vocab)) +print(tokenizer.tokenize(sentence)) + +# (2) : 다양한 언어를 담고 있는 다른 데이터에서 학습한 모델인 'bert-base-multilingual-uncased'의 Tokenizer를 가져와 Tokenization \ No newline at end of file diff --git a/Pytorch/3_NLP/Frequency_Based.png b/Pytorch/3_NLP/Frequency_Based.png new file mode 100644 index 0000000..c45b79a Binary files /dev/null and b/Pytorch/3_NLP/Frequency_Based.png differ diff --git a/Pytorch/3_NLP/README.md b/Pytorch/3_NLP/README.md new file mode 100644 index 0000000..010eeae --- /dev/null +++ b/Pytorch/3_NLP/README.md @@ -0,0 +1,165 @@ +# NLP +Text 데이터를 분석하고 모델링하는 분야를 자연어 처리(NLP)라 함
+- NLP(Natural Language Processing, NLP): 자연어 처리 + - NLU(Natural Language Understanding, NLU): 자연어 이해 + - Text -> Meaning + - NLG(Natural Language Generation, NLH): 자연어 생성 + - Meaning -> Text + +
+ +torchtext를 이용해 데이터셋 불러오기 +``` +from torchtext import data +from torchtext import datasets + +# Data Setting +TEXT = data.Field(lower=True, batch_first=True) +LABEL = data.Field(sequential=False) + +train, test = datasets.IMDB.splits(TEXT, LABEL) +``` + +## 문자를 숫자로 표현 +문자를 단순하게 숫자로 바꿔버리는 경우 Input에서 공통적인 패턴(Pattern) 또는 특징(Feature)을 찾기 힘듬
+모델을 잘 만들기 위해서는 Input의 공통적 패턴을 잘 찾아내도록 해줘야 함 + +단순하게 문자를 숫자로 바꿔 버리게 되면 몇 가지 문제가 생김
+- 내가 만든 사전에 없는 단어가 나오는 경우 +- '철수'나 '철수에게'는 같은 단어 같은데 조사 차이로 나눠야 하는가? +- 이중적 의미의 단어(예] 사과)를 같게 처리하는 경우 + +### Corpus & Out-of-Vocabulary(OOV) +Token을 저장해둔 Vocabulary에 Token이 없어서 처음 본 Token이 나오는 현상을 Out-of-Vocabulary(OOV)라 함
+이런 상황을 대비해 특수한 Token을 만들어 사용(
++ +BPE Algorithm +1. 단어 횟수를 기록한 사전을 만듬(띄어쓰기 기반의 Tokenization), 이때 사전의 단어 글자는 모두 띄어 표현 +2. 각 단어에 대해 연속된 2개의 글자의 숫자를 세어 가장 많이 나오는 글자 2개의 조합을 찾음(Character bi-gram) +3. 두 글자를 합쳐 기존의 사전의 단어를 수정 +4. 미리 정해 놓은 횟수만큼 2~3번의 과정을 반복 + +- [CODE] : BPE.py +- [CODE] : Sentencepiece.py +- [CODE] : BertTokenizer.py + +## Word Embedding +문자를 그대로 숫자로 표현하게 되면 딥러닝 모델 학습에 문제가 생김 + +1 + 1 = 2 이지만 1이 '책상'을 의미하고 2가 '위에'를 의미한다면 그렇다고 해서 '책상' 두개가 '위에'를 의미하지는 않음 (1 + 1 = 2) + +이와 같은 문제가 일어나는 이유는 Token을 유한한 개수를 가진 변수로 생각하고 범주형 자료(Categorical Data)로 바로 표현했기 때문 + +==> 이러한 문제를 해결하기 위해 범주형 자료 변수를 표현하는 방법 중 한가지가 바로 원-핫 인코딩(통계학에서는 가변수(Dummy Variables)라고 표현) + +### 원-핫 인코딩 +원-핫 인코딩을 위해선 Corpus를 모두 Tokenization해 Vocabulary를 만들고 각 Token마다 Index를 정해야 함 + +
++ +- [CODE] one_hot_encoding.py + +원-핫 인코딩 말고도 Frequency-based Method와 Dense Embedding Method로 나눠 볼 수 있음 + +### Frequency-based Method +단어의 횟수를 기반으로 표현하는 방식(Frequency-based Method)은 말 그대로 문장에 있는 Token의 등장 횟수를 세어 표현하는 방식 + +Token의 횟수에만 집중하기 때문에 주머니 같은 곳에 Token을 모아놓고 단어를 뽑아 문장을 표현한다해서 **Bag of Words(BoW)** 라 표현 + +표현 방법은 Token을 원-핫 인코딩한 결과를 문장마다 합하면 쉽게 나타낼 수 있음 + +
++ +원-핫 인코딩을 거쳐가지 않고 {key: token, value: count} 형식으로 문장이나 문단의 Token을 직접 세어 표현가능 + +Frequency-based Method 방법에는 한 가지 문제가 있는데 영어의 "a", "an", "of"와 같은 특정 token은 문장이나 문단과 상관없이 많이 등장할 수 밖에 없음. 따라서 해당 Token의 수치는 매우 높은 값을 갖게됨 --> 등장 빈도가 적은 Token에 나쁜 영향을 미칠 수밖에 없음 + +==> 이러한 문제를 해결하기 위한 방법으로 **TF-IDF(Term Frequency - Inverse Document Frequency)** 라는 표현 방법이 존재 + +But, Frequency-Based 방법 또한 문제가 있는데 단어의 순서가 무시된다는 점. + +우리가 문장을 구성할때 특정 Token이 나올 확률은 이전 Token에 영향을 받을 수 밖에 없음. 하지만 횟수 기반의 표현 방식은 Token의 순서 정보를 전혀 담을 수 없어 NLG와 같이 새로운 문장을 생성해야 하는 Task에서 사용이 어려움 + +### Dense Representation +원-핫 인코딩을 이용한 문자 표현의 문제점 중 한 가지는 변수의 희소성. + +모든 Token의 개수인 V만큼의 벡터를 만드는 것뿐 아니라 대부분 0이면서 극소수의 위치에만 값을 가지고 있기 때문에 상당히 큰 메모리 사용량일 필요하지만 대부분은 0으로 돼있어 비효율적(파이썬의 sparse 라이브러리를 이용해 계산이나 메모리 사용을 효율화할 순 있음) + +### Word2vec: idea +Word2vec 방법은 vector("King") - vector("Man") + vector("Women") = vector("Queen") 이라는 예제를 이용해 관계를 연산으로 설명할 수 있는 벡터표현이 가능하다는 것을 보여 많은 관심을 갖게 됨 + +먼저 Word2vec 모델 학습에는 'Token의 의미는 주변 Token의 정보로 표현된다' 라 가정 + +==> 특정 Token을 기준으로 주변에 비슷한 Token이 있다면 해당 Token은 비슷한 위치의 벡터로 표현되도록 학습 + +Word2vec 학습 과정을 **CBOW(Continuous Bag-of-Words Model)** 와 **Skip-Gram** 이 두가지 방법 모두 공통적으로는 문장을 윈도우 형태로 부분만 보는 것을 기본으로 함 + +기준 Token의 양옆 Token을 포함한 윈도우가 이동하면서 윈도우 속 Token과 기준 Token의 관계를 학습시키는 과정을 진행 + +주변 Token을 **Context** , 기준 Token을 **Target** 이라 표현 + +- CBoW + - Context Token의 벡터로 변환해 더한 후 Target Token을 맞춤 +- Skip-Gram + - Target Token을 벡터로 변환 후 Context Token을 맞춤 + + +모델 구조 +
+
++ +#### 학습 과정 diff --git a/Pytorch/3_NLP/Sentencepiece.py b/Pytorch/3_NLP/Sentencepiece.py new file mode 100644 index 0000000..3aead99 --- /dev/null +++ b/Pytorch/3_NLP/Sentencepiece.py @@ -0,0 +1,6 @@ +# https://github.com/google/sentencepiece + +import sentencepiece as spm +s = spm.SentencePieceProcessor(model_file='spm.model') +for n in range(5): + s.encode('New York', out_type=str, enable_sampling=True, alpha=0.1, nbest=-1) \ No newline at end of file diff --git a/Pytorch/3_NLP/one_hot_encoding.png b/Pytorch/3_NLP/one_hot_encoding.png new file mode 100644 index 0000000..4fb23e2 Binary files /dev/null and b/Pytorch/3_NLP/one_hot_encoding.png differ diff --git a/Pytorch/3_NLP/one_hot_encoding.py b/Pytorch/3_NLP/one_hot_encoding.py new file mode 100644 index 0000000..8f787ea --- /dev/null +++ b/Pytorch/3_NLP/one_hot_encoding.py @@ -0,0 +1,41 @@ +S1 = "나는 책상 위에 사과를 먹었다" +S2 = "알고 보니 그 사과는 Json 것이었다" +S3 = "그래서 Jason에게 사과를 했다" + +token2idx = {} +index = 0 + +for sentence in [S1, S2, S3]: + tokens = sentence.split() + for token in tokens: + if token2idx.get(token) == None: + token2idx[token] = index + index += 1 + +print(token2idx) +print('\n') + +V = len(token2idx) + +token2vec = [([0 if i != idx else 1 for i in range(V)], idx, token) for token, idx in token2idx.items()] +for x in token2vec: + print("\t".join([str(y) for y in x])) +print('\n') + +# python numpy를 이용해 문장을 원-핫 인코딩으로 바꾸는 방법 +import numpy as np + +for sentence in [S1, S2, S3]: + onehot_s = [] + tokens = sentence.split() + for token in tokens: + if token2idx.get(token) != None: + vector = np.zeros((1,V)) + vector[:,token2idx[token]] = 1 + onehot_s.append(vector) + else: + print("UNK") + + print(f"{sentence}: ") + print(np.concatenate(onehot_s, axis = 0)) + print('\n') \ No newline at end of file diff --git a/Pytorch/3_NLP/word2vec_model.png b/Pytorch/3_NLP/word2vec_model.png new file mode 100644 index 0000000..3816634 Binary files /dev/null and b/Pytorch/3_NLP/word2vec_model.png differ diff --git a/README.md b/README.md index 7c5ce19..342161c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,34 @@ # DeepLearning 요약 -DeepLearning을 연구하고 학습해오며 배웠던 것들을 정리하는 공간. +DeepLearning을 학습해오며 배웠던 것들을 정리하는 공간. ## Structure - Theory : 이론 위주 학습 정리 - Pytorch : Pytorch를 이용해 신경망 구현 - Python : Python으로 신경망 구현 +- TensorFlow : Keras Tensorflow로 신경망 구현 + +## Deep Learning 전체 프로세스 +
+## Index +- 딥러닝 모델 학습 전 알아두어야 할 이론 + - [확률적 경사 하강법(SGD) 이란?](Theory/02_StochasticGradientDescent/) + - [역전파법 이란?](Theory/03_Backpropagation/) + - [역전파법 구현 - Python](Python/01_Backpropagation_Theory/) + - [텐서, 손실함수, 옵티아미저란?](TensorFlow/0_dl_math/) +- 앞먹임 신경망 + - [앞먹임 신경망 이론(Feedforward Neural Network) 이란?](Theory/01_FeedforwardNeuralNetwork/) + - [앞먹임 신경망 구현 - Keras](TensorFlow/1_BasicNeuralNet/) + - [앞먹임 신경망 구현 - Pytorch](Pytorch/1_MLP/) +- 자기부호화기 + - [자기부호화기(AutoEncoder) 란?](Theory/04_AutoEncoder/) + - [자기부호화기(AutoEncoder) 구현 - Pytorch](Pytorch/1_MLP/1_AutoEncoder.ipynb) +- 합성곱 신경망 + - [합성곱 신경망(CNN) 이란?](Theory/05_CNN/) + - [CNN 구현 - Keras](TensorFlow/2_CNN/) + - [CNN 구현 - Pytorch](Pytorch/2_CNN/) +- 재귀 신경망 + - [재귀 신경망(RNN) 이란?](Theory/06_RNN/) + - [재귀 신경망 구현 - Keras](TensorFlow/3_RNN/) + - [재귀 신경망 구현 - Pytorch](Pytorch/3_NLP/) \ No newline at end of file diff --git a/TFLite/SpeechRecognition/aos_speech_commands/.gitignore b/TFLite/SpeechRecognition/aos_speech_commands/.gitignore new file mode 100755 index 0000000..23a894d --- /dev/null +++ b/TFLite/SpeechRecognition/aos_speech_commands/.gitignore @@ -0,0 +1,16 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild + +/.gradle/ +/.idea diff --git a/TFLite/SpeechRecognition/aos_speech_commands/app/.gitignore b/TFLite/SpeechRecognition/aos_speech_commands/app/.gitignore new file mode 100755 index 0000000..23a894d --- /dev/null +++ b/TFLite/SpeechRecognition/aos_speech_commands/app/.gitignore @@ -0,0 +1,16 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild + +/.gradle/ +/.idea diff --git a/TFLite/SpeechRecognition/aos_speech_commands/app/build.gradle b/TFLite/SpeechRecognition/aos_speech_commands/app/build.gradle new file mode 100755 index 0000000..907fd6b --- /dev/null +++ b/TFLite/SpeechRecognition/aos_speech_commands/app/build.gradle @@ -0,0 +1,53 @@ +apply plugin: 'com.android.application' +apply plugin: 'kotlin-android-extensions' +apply plugin: 'kotlin-android' +apply plugin: 'de.undercouch.download' + +android { + compileSdkVersion 28 + buildToolsVersion '28.0.3' + defaultConfig { + applicationId "com.google.tflite.speechrecognition" + minSdkVersion 21 + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + } + } + aaptOptions { + noCompress "tflite" + } + compileOptions { + sourceCompatibility = '1.8' + targetCompatibility = '1.8' + } +} + +// import DownloadModels task +project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets' +project.ext.TMP_DIR = project.buildDir.toString() + '/downloads' + +// Download default models; if you wish to use your own models then +// place them in the "assets" directory and comment out this line. +//apply from: "download_model.gradle" + +apply from:'download_model.gradle' + + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation 'com.android.support:appcompat-v7:28.0.0' + implementation 'com.android.support.constraint:constraint-layout:1.1.3' + implementation 'com.android.support:design:28.0.0' + implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly' + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" +} +repositories { + mavenCentral() +} diff --git a/TFLite/SpeechRecognition/aos_speech_commands/app/download_model.gradle b/TFLite/SpeechRecognition/aos_speech_commands/app/download_model.gradle new file mode 100755 index 0000000..7d3d49d --- /dev/null +++ b/TFLite/SpeechRecognition/aos_speech_commands/app/download_model.gradle @@ -0,0 +1,26 @@ + +task downloadZipFile(type: Download) { + src 'http://storage.googleapis.com/download.tensorflow.org/models/tflite/conv_actions_tflite.zip' + dest new File(buildDir, 'zips/') + overwrite true +} + + +task downloadAndUnzipFile(dependsOn: downloadZipFile, type: Copy) { + from zipTree(downloadZipFile.dest) + into project.ext.ASSET_DIR +} + + +task extractModels(type: Copy) { + dependsOn downloadAndUnzipFile +} + +tasks.whenTaskAdded { task -> + if (task.name == 'assembleDebug') { + task.dependsOn 'extractModels' + } + if (task.name == 'assembleRelease') { + task.dependsOn 'extractModels' + } +} \ No newline at end of file diff --git a/TFLite/SpeechRecognition/aos_speech_commands/app/proguard-rules.pro b/TFLite/SpeechRecognition/aos_speech_commands/app/proguard-rules.pro new file mode 100755 index 0000000..f1b4245 --- /dev/null +++ b/TFLite/SpeechRecognition/aos_speech_commands/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/TFLite/SpeechRecognition/aos_speech_commands/app/src/androidTest/java/com/google/tflite/speechrecognition/ExampleInstrumentedTest.java b/TFLite/SpeechRecognition/aos_speech_commands/app/src/androidTest/java/com/google/tflite/speechrecognition/ExampleInstrumentedTest.java new file mode 100755 index 0000000..ca91cff --- /dev/null +++ b/TFLite/SpeechRecognition/aos_speech_commands/app/src/androidTest/java/com/google/tflite/speechrecognition/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package com.google.tflite.speechrecognition; + +import android.content.Context; +import android.support.test.InstrumentationRegistry; +import android.support.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getTargetContext(); + + assertEquals("com.google.tflite.speechrecognition", appContext.getPackageName()); + } +} diff --git a/TFLite/SpeechRecognition/aos_speech_commands/app/src/main/AndroidManifest.xml b/TFLite/SpeechRecognition/aos_speech_commands/app/src/main/AndroidManifest.xml new file mode 100755 index 0000000..daee6e0 --- /dev/null +++ b/TFLite/SpeechRecognition/aos_speech_commands/app/src/main/AndroidManifest.xml @@ -0,0 +1,24 @@ + +
++ +## 합성곱 +- 그레이 스케일 이미지를 예로 이미지 사이즈를 (W x W), 픽셀 인덱스를 (i, j)로 나타낸다 했을 때 픽셀 값을 x(i,j) +- 필터(filter)라는 작은 크기의 이미지를 상정하고 크기를 (H x H), 필터의 픽셀은 인덱스 (p, q)로 나타내고 픽셀 값은 h(p,q) +- 이미지의 합성곱이란 이미지와 필터 사이 아래와 같은 연산 + +
++ +### 합성곱의 작용 +이미지의 합성곱(또는 상관)은 필터의 명암 패턴과 유사한 명암 패턴이 입력된 이미지 어디에 있는지를 검출하는 작용을 함 + +### 패딩 +- 합성곱은 이미지에 필터를 겹쳤을 때 이미지와 필터가 겹쳐지는 픽셀끼리 곱을 구하여 필터 전체에 그 값을 합하는 연산 + - 이미지로부터 필터가 벗어나는 위치에 적용하는 것은 불가능 +- 합성곱 계산의 결과 이미지 크기는 입력 이미지의 크기보다 작아지게 됨 + - (W - 2[H/2]) x (W - 2[H/2]) + - [.] : 소수점은 버림 + - ex] 8x8 입력이미지, 3x3 필터 : 8-2[3/2] = 6 +- 합성곱한 결과 이미지가 입력 이미지와 크기가 같으면 좋음 + - 입력 이미지의 바깥쪽 테두리를 만들어 크기를 늘려서 출력 이미지의 크기가 원래 입력 이미지와 같은 크기가 되도록 함 + - 이 태두리 부분은 픽셀값을 0으로 설정하는 방법으로 제로 패딩(zero-padding)이라 부르는 방법을 사용 + - CNN에서 제로 패딩이 널리 쓰이나 이미지 처리라는 관점에서 보면 좋지 않음 + - 제로 패딩 처리로 인해 출력 이미지의 주변부가 어둡게 되기 때문 + - 테두리 부분의 픽셀값을 0이 아닌 더 그럴듯한 값으로 채우는 기법이 존재 + +### 스트라이드 +- 필터가 적용되는 위치를 1픽셀이 아닌 여러 픽셀씩 움직이며 계산할 수도 있음 +- 이 필터의 적용 위치 간격을 **스트라이드(stride)** 라 함 +- 출력 이미지의 크기 + - ([(W-1/s)] + 1) x ([(W-1/s)] + 1) +- 크기가 큰 입력 이미지를 다룰 때 합성곱층의 출력쪽 유닛 수가 너무 많아지는 것을 막기 위해 2 이상의 스트라이드 값을 적용하는 경우가 존재 + - But, 스트라이드 값을 크게 잡는 것은 이미지의 특징을 놓칠 가능성을 의미 일반적으로 성능이 좋아지지 않음 + +## 합성곱층 +- 합성곱 연산을 하는 단층 신경망 +- 채널 수가 K인 이미지의 각 픽셀은 K개의 값을 가짐 + - RGB 3가지 색으로 구성된 이미지의 채널 수는 3 +- CNN은 이런 이미지를 입력으로 받지만 대개 중간층(합성곱층과 폴링층)에서 그 이상의 채널 수(K=16, 256 등)을 다룸 +- 이러한 중간층의 출력을 이미지라 부르지 않고 **맵(map)** 이라 부르기도 함 + +
++ +- 각 필터는 입력 이미지와 같은 채널 수 K를 가짐 +- m = 0, 1, 2의 각 필터 m에 대하여 병렬로 계산이 실행 각각 한 개 채널이 출력 +- 입력 이미지의 채널 수와 상관없이 하나의 필터로부터 나오는 출력은 항상 1채널이 됨 +- 합성곱층의 최종 출력은 필터 수 M과 같은 수의 채널 수를 갖는 다 채널 이미지가 됨 + - 입력의 크기가 W x W x K 일 때 (합성곱층의 필터수는 M) 출력의 크기는 W x W x M이 됨 +- 가중치는 출력층의 같은 채널에 속하는 모든 유닛에서 같음(**가중치 공유**) +- 합성곱층의 가중치는 필터 그 자체이므로 최적화의 대상이 필터가 됨, 역전파법에 의해 합성곱층의 델타와 필터의 오차 기울기가 계산 + +## 폴링층 +- 폴링층은 보통 합성곱층의 바로 뒤에 배치 +- W x W x K인 입력 이미지의 픽셀을 중심으로 H x H 크기의 정사각형 영역을 잡아 채널 k마다 독립적으로 H^2개 있는 픽셀값을 사용하여 하나의 픽셀값을 구함. 이 값을 구하는 방법으로 아래의 2가지 방법이 있음 + - 최대폴링 + - 평균폴링 +- 폴링의 계산은 입력 이미지의 각 채널마다 병렬로 실행됨 보통 폴링층의 출력 채널 수는 입력 이미지의 채널 수와 일치 +- 합성곱층과 마찬가지로 폴링층에도 2 이상의 스트라이드를 설정할 수 있음 +- 폴링층의 유닛에 활성화 함수를 적용하는 것은 가능은 하나 보통은 적용하지 않음 +- 폴링층에는 학습에 따라 변화할 수 있는 파라미터가 존재하지 않음 + +## 정규화층 +### 국소 콘트라스트 정규화 +- 자연물 이미지를 입력으로 하는 이미지 인식 문제에서 입력 이미지의 전체적인 밝기나 대비의 차이를 잘 흡수해야 함 +- 주변의 조명에 따라 이미지 전체의 밝기나 대비가 크게 변하게됨 +- 이러한 변화에도 불구하고 이미지를 잘 인식하기 위해 이미지의 명암을 보종의 방법으로 정규화하는 방법이 필요 +- 정규화 방법 + - 이미지의 집합(훈련 데이터)에 대한 통계치를 이용하는 처리 + - 국소 콘트라스트 정규화(local contrast normalization) + - 감산 정규화(subtractive normalization) + - 제산 정규화(divisive normalization) +- 국소 콘트라스트 정규화 또한 가중치는 고정값이어서 폴링층과 마찬가지로 학습이 가능한 파라미터는 존재하지 않음 \ No newline at end of file diff --git a/Theory/06_RNN/README.md b/Theory/06_RNN/README.md new file mode 100644 index 0000000..9e5851c --- /dev/null +++ b/Theory/06_RNN/README.md @@ -0,0 +1,63 @@ +# RNN +재귀 신경망(RNN)은 음성이나 언어, 동영상과 같은 연속열 데이터를 다루는 신경망 + +- 데이터는 일반적으로 그 길이가 샘플마다 무척 다르며 연속열 내의 요소의 순서(문맥)에 의미가 있는 것이 특징 + +## 연속열 데이터의 분류 +- 연속열 데이터란 각각의 요소에 순서가 있는 모임으로 음성, 동영상, 텍스트 등이 있음 + +## RNN의 구조 +- 재귀 신경망(RNN)은 내부에 (방향이 있는) 순환경로를 가진 신경망을 통틀어 부르는 말로 그러한 구조 덕에 정보를 일시적으로 기억하고 그에 따라 반응을 동적으로 변화 가능 +- 그로 인해 연속열 데이터 안에 존재하는 '문맥'을 포착하고 분류 문제도 잘 처리 가능 +- RNN의 동작은 각각의 시각 t마다 하나의 입력 X^t를 입력받아 또 동시에 하나의 출력 y^t를 출력하는 것 +- 신경망 내부에 있는 귀환로에 의해 출력을 계산할 때 RNN이 이전에 입력받은 모든 입력(입력의 히스토리)이 관여 +- RNN은 이론상 과거의 모든 입력에 대해 하나의 출력을 내놓는 매핑을 모형화 + +
++ +- 연속열 데이터 전체가 일괄로 주어진 경우 그 연속열 데이터를 역순으로 RNN에 입력하는 것도 가능 +- 원래 순서대로 연속열을 입력받는 RNN과 역순으로 연속열을 입력받는 RNN을 만들어 양쪽의 출력층을 통합해 구성한 신경망을 **양방향 RNN(bidirectional RNN)** 이라 함 + - 양방향 RNN이 정방향 입력만 주어지는 RNN보다 성능이 더 좋다고 알려짐 + +## 순전파 계산 +- RNN의 귀환로는 중간층의 출력을 스스로의 입력으로 되돌리는 것으로 이 사이의 모든 유닛 간에 결합이 존재 +- 이 귀환로로 들어오는 입력이 한 시각 전의 신호로 시각 t에 대한 중간층 각 유닛의 입력은 같은 시각에 입력층으로부터 전해지는 것과 시각 t - 1 에 중간층에서 나온 출력이 피드백된 것의 합 + +## 역전파 계산 +- RNN을 학습하는 데는 앞먹임 신경망과 같이 SGD를 사용, 어떤 경우든 각 층의 가중치에 대한 오차의 미분을 계산 + - **RTRL(RealTime Recurrent Learning)** + - 메모리 효율이 좋음 + - **BPTT(BackPropagation Through Time)** + - 계산속도가 빠르며 간단 + +- BPTT는 RNN을 시간 방향으로 펼쳐 앞먹임 신경망과 같은 형태로 만든 후 역전파 계산을 수행 + - 시간 방향으로 펼친다는 것은 매 시각마다의 RNN의 각 층이 별개인 것처럼 간주 + +
++ +## 장단기기억 +### RNN의 기울기 소실 문제 +- RNN의 경우 포착할 수 있는 문맥의 길이, 즉 현재 시각으로부터 얼마나 먼 과거의 입력까지 반영할 수 있는지가 중요 + - 이론상으론 과거의 모든 입력 이력이 고려되어야 할테지만 실제 RNN의 출력에 반영시킬 수 있는 것은 기껏해야 과거 10시각 정도로 알려져 있음 +- 이러한 한계는 앞먹임 신경망의 기울기 소실 문제와 같은 원인에서 비롯 +- 층수가 많은 신경망에는 역전파법으로 기울기를 계산시 층을 거슬러 올라감에 따라 기울기 값이 폭발적으로 커지거나 0으로 소멸해 버리기 쉬운 성질 존재 + - RNN의 역전파 계산은 시간 방향으로 전개되므로 층수가 매우 많은 앞먹임 신경망의 역전파 계산으로 바꿔 생각할 수 있음 ==> RNN의 본래 층수가 적다 해도 역전파 계산 시에는 많은 층수를 다루는 것과 마찬가지 그에 따라 기울기 값도 발산하거나 소멸하거나 둘 중 하나가 되기 쉬움 +- RNN으로는 단기적 기억은 실현할 수 있지만 더 장기적 기억을 실현하는 것은 어려움 + +## LSTM의 개요 +RNN으론 장기적 기억 실현이 어려워 장기적 기억을 실현하기 위해 제안된 방법 중 가장 성공적 시도가 장단기기억(Long Short-Term Memory) +- LSTM은 RNN에 비해 중간층의 각 유닛이 메모리 유닛이라 불리는 요소로 구성된 구조로 그 외의 구조는 기존 RNN과 같음 + +## 입력과 출력의 연속열 길이가 다른 경우 +### 은닉 마르코프 모델 +- RNN(LSTM 포함)은 각 시각 t마다 하나의 입력 x^t를 받아 하나의 출력 y^t를 내놓음 + - 이 동작은 입력 연속열과 다른 길이를 가진 연속열을 추정해야 하는 경우엔 적합하지 않음 +- **은닉 마르코프 모델(Hidden Markov Model, HMM)** 은 이러한 문제를 다루는 가장 일반적 방법 +- HMM은 내부 상태를 숨겨진 변수로 가지고 이 변수가 시각과 함께 확률적으로 변화 동시에 HMM은 현재의 내부 상태에 기ㅗ한 확률적 관측을 생성 + +## 커넥셔니스트 시계열 분류(CTC) +- Connectionist Temporal Classification +- 입력과 출력 사이에 연속열의 길이가 서로 다른 경우의 분류 문제를 HMM을 쓰지 않고 신경망만을 사용해 해결하려는 방법 +- CTC는 RNN의 출력에 대한 해석을 바꾸어 입력 연속열과 길이가 다른 출력 연속열을 다룰 수 있도록 해줌 \ No newline at end of file diff --git a/Theory/06_RNN/RNNBasic.png b/Theory/06_RNN/RNNBasic.png new file mode 100644 index 0000000..f7a421a Binary files /dev/null and b/Theory/06_RNN/RNNBasic.png differ diff --git a/Theory/06_RNN/RNN_Backpropagation.png b/Theory/06_RNN/RNN_Backpropagation.png new file mode 100644 index 0000000..11a4856 Binary files /dev/null and b/Theory/06_RNN/RNN_Backpropagation.png differ diff --git a/Transformer/01_TextClassification/README.md b/Transformer/01_TextClassification/README.md new file mode 100644 index 0000000..c95579c --- /dev/null +++ b/Transformer/01_TextClassification/README.md @@ -0,0 +1,213 @@ +# Text Classification using DistilBERT +Huggingface의 DistilBERT(BERT와 비교해 성능은 비슷하지만 훨씬 작고 효율적) 모델을 사용해 6개의 감정 모델을 분류
+Huggingface 생태계의 핵심 라이브러리 사용 +- **데이터셋(Dataset)** +- **토크나이저(Tokenizer)** +- **트랜스포머스(Transformers)** + +## 1. 데이터셋 +### 허깅페이스 데이터셋 사용 +- **list_datasets()** 함수를 사용하면 허브에서 제공하는 데이터셋 목록이 출력됨 +- **load_dataset()** 함수로 데이터셋 로드 + +### Huggingface hub에 데이터셋이 없는 경우 +- 데이터셋은 로컬 데이터셋이나 원격 데이터셋에 사용 가능한 로딩 스크립트를 제공해줌 +- 포멧에 따른 데이터셋 로딩 방법 + - CSV: load_dataset("csv", data_files="my_file.csv") + - text: load_dataset("text", data_files="my_file.txt") + - JSON: load_dataset("json", data_files="my_file.jsonl") +- 원격 데이터 로딩 방법 +``` +dataset_url = "https://www.~/train.txt" +!wget {dataset_url} +emotions_local = load_dataset("csv", data_files="train.txt", sep=";", names=["text", "label"]) +``` +구분자(sep)와 열 이름(names)을 지정 +``` +dataset_url = "https://www.~/train.txt" +emotions_local = load_dataset("csv", data_files=dataset_url, sep=";", names=["text", "label"]) +``` + +### 데이터셋 to 데이터프레임 +- Dataset 객체를 판다스 DataFrame으로 변환하게 되면 고수준 데이터 시각화 API 등 편리한 API를 사용할 수 있음 +- 데이터셋은 Dataset의 출력 포맷을 변경하는 **set_format()** 메서드를 제공 + +### 클래스 분포 확인 +- 텍스트 분류 문제를 다룰땐 샘플의 클래스 분포 조사를 해야 함 +- 클래스 분포가 편향된 데이터 셋은 훈련 손실과 평가 지표 측면에서 균형 잡힌 데이터셋과 다른 처리법이 필요 +- pandas와 matplot을 이용하면 클래스 분포를 시각화할 수 있음 + +- 데이터셋이 불균형 한 경우 처리방법 + - 소수 클래스를 랜덤하게 오버샘플링(oversampling) + - 다수 클래스를 랜덤하게 언더샘플링(undersampling) + - 클래스의 대표성이 부족하다면 레이블된 데이터를 더 많이 수집 + +- 훈련/테스트 분할을 만들기 전에는 샘플링 전략을 사용하면 안됨 그렇지 않으면 분할 사이에 많은 정보가 새어나감 + - 일반적으로 훈련 세트에만 샘플링 전략을 사용 +- 불균형한 클래스 분포에서의 샘플링 기법: https://imbalanced-learn.org/stable/ + +### 길이 확인 +- 트랜스포머 모델은 **최대 문맥 크기(maximum context size)** 라는 최대 입력 시퀀스 길이가 존재 + - DistilBERT를 사용하는 애플리케이션에서 최대 문맥 크기는 512 토큰 + +## 2. 토큰나이저 +- 토큰화란 문자열을 모델이 사용하는 기본 단위로 분할하는 단계 + +### 문자 토큰화 +- 가장 간단한 토큰화 방법으로 각 문자를 개별로 모델에 주입하는 것 +- 텍스트를 숫치 데이터로 변환한 후 one-hot vector의 2D 텐서로 변경 +- one-hot vector는 machine learning에서 순서형(ordinal) 또는 명목형(nominal) 범주 데이터를 인코딩 하기 위해 자주 사용 + - 텍스트를 단순히 숫자로만 변경할시 이름 사이에 가상의 순서 또는 상관없는 관계가 생길수 있음 +- 문자 토큰화는 철자 오류, 희귀단어 처리 등 장점이 있지만 단어 같은 언어 구조를 이 데이터에서 학습해야 한다는 큰 단점이 존재 + - 문자 수준의 토큰화는 거의 사용되지 않음 + +### 단어 토큰화 +- 텍스트를 문자가 아닌 단어로 분할, 단어를 정수로 매핑 +- 처음부터 단어를 사용하면 모델이 문자에서 단어를 학습하는 단계가 생략되어 훈련 과정의 복잡도가 감소함 +- 단어 토큰화는 단어에 곡용, 활용형, 철자 오류가 포함되어 어휘사전이 금세 수백만 개까지 늘어나는 단점이 존재 + - 어휘 사전이 크면 신경망의 파라미터 역시 많이 필요해짐 + - 어휘사전의 크기를 제한하는 일반적 방법은 드물게 등장하는 단어는 무시 + - ex)말뭉치에서 자주 등장하는 10만개 단어만 사용 + - but, 그렇게 되면 단어 토큰화 과정에서 중요 정보를 일부 잃게 됨 + +### 부분단어 토큰화 +- 부분단어 토큰화는 문자 토큰화와 단어 토큰화의 장점을 결합 +- 드물게 등장하는 단어를 더 작은 단위로 나누면 모델이 복잡한 단어나 철자 오류를 처리하기 용이 +- 입력 길이를 적절한 크기로 유지하기 위해 자주 등장하는 단어를 고유 항목으로 유지 +- NLP 에서 많이 사용되는 부분단어 토큰화는 BERT와 DistilBERT의 토큰나이저인 **WordPiece** +- 트랜스포머스는 사전 훈련된 모델에 연관된 토크나이저를 빠르게 로드하는 **AutoTokenizer** 클래스 제공 + - 이 클래스의 **from_pretrained()** 메서드를 허브의 모델ID나 로컬 파일 경로와 함께 호출 +- AutoTokenizer 클래스는 체크포인트 이름을 사용해 모델의 설정, 사전 훈련 가중치, 어휘 사전을 자동으로 추출하는 자동 클래스 + +## 3. 텍스트 분류 모델 훈련 +- DistilBERT 모델은 텍스트 시퀀스에 있는 마스킹된 단어를 예측하도록 사전 훈련, 이 모델을 바로 텍스트 분류에 사용하지는 못함 + +
++ + - 토큰 인코딩(Token Encoding) + - 텍스트를 토큰화 하여 Token-Encoding이라 부르는 one-hot 벡터로 나타냄 + - Tokenizer 어휘 사전의 크기(2만 ~ 2백만)가 토큰 인코딩의 차원을 결정 + - 토큰 임베딩(Token Embedding) + - 토큰 인코딩을 저차원 공간의 벡터인 토큰 임베딩으로 변환 + - 토큰 임베딩을 Encoder 블록 층에 통과시켜 각 입력 토큰에 대한 은닉 상태를 생성 + - 각 은닉 상태는 언어 모델링의 사전 훈련 목표(DistilBERT의 경우 마스킹된 토큰)을 달성하기 위해 마스킹된 입력 토큰을 예측하는 층으로 보냄 + +- 텍스트 분류 모델 훈련 방법 + - 특성 추출 + - 사전 훈련된 모델을 수정하지 않고 은닉 상태를 특성(feature)으로 사용해 분류 모델을 훈련 + - 미세 튜닝 + - 사전 훈련된 모델의 파라미터도 업데이트하기 위해 전체 모델을 엔드-투-엔드로 훈련 + +### 트랜스포머를 특성 추출기로 사용 +- 바디의 가중치를 동결하고 은닉 상태를 분류 모델의 특성으로 사용 +- 이 방식은 작거나 얕은 모델을 빠르게 훈련한다는 장점이 존재 + +### 사전 훈련된 모델 사용 +- 트랜스포머스의 자동 클래스인 **AutoModel**을 사용 +- AutoModel 클래스 또한 사전 훈련된 모델의 가중치를 로드하는 **from_pretrained()** 메서드 존재 +- AutoModel은 토큰 인코딩을 임베딩으로 변환한 다음 인코더 스택에 통과시켜 은닉 상태를 반환 +- 은닉 상태 텐서의 크기는 [batch_size, n_tokens, hidden_dim] + - ex) [1, 6, 768]: 6개의 입력 토큰마다 768차원의 벡터가 반환 +- 분류 작업에선 보통 [CLS] 토큰에 연관된 은닉 상태를 입력 특성으로 사용 + - BERT모델에 있는 [CLS]토큰은 분류 작업에서 전체 시퀀스의 정보가 담긴 특수 토큰 + +### 훈련 세트 시각화 +- UMAP 알고리즘을 사용해 벡터를 2D로 투영 + - UMAP은 특성이 [0,1] 범위에 놓일 때 잘 동작함 + - Scikit-learn의 MinMaxScaler를 적용 + +## 트랜스포머 미세 튜닝 +- 미세튜닝방식에서는 은닉 상태를 고정된 특성으로 사용하지 않고 전체 모델을 훈련 +- 그렇기에 분류 헤드는 미분 가능해야 함(미분이 가능해야 역전파법으로 weight를 구할 수 있기 때문) +- 트랜스포머스의 **Trainer** API를 사용 +- AutoModel 클래스 대신 AutoModelForSequenceClassification을 사용 + - 사전 훈련된 모델 출력위에 베이스 모델과 함께 쉽게 훈련할 수 있는 분류헤드가 존재 +- 훈련 파라미터를 정의하기 위해 **TrainingArguments** 클래스를 사용 + - 많은 정보를 저장하며 훈련과 평가를 상세히 제어 가능 + - TrainingArguments Parameter + - output_dir: Output Directory, 모델의 예측과 체크포인트를 저장해두는 장소 + - num_train_epochs: epochs 횟수 + - learning_rate: 학습률 + per_device_train_batch_size/ per_device_eval_batch_size: 배치크기 + - weight_decay: 가중치 감쇠 정도 + - evaluation_strategy: Training중 채택할 평가 전략 + - "no" : 학습하는 동안 평가 하지 않음 + - "steps" : eval_steps 마다 평가 + - "epoch" : epoch 당 평가 + - logging_steps: 업데이트 단계 수 + - save_strategy: 체크 포인트 저장 전략 + - "no": 저장하지 않음 + - "epoch": epoch 마다 저장 + - "steps": save_step마다 저장 + - load_best_model_at_end: 학습이 끝났을때 가장 best model을 load + - 참고: https://huggingface.co/docs/transformers/v4.24.0/en/main_classes/trainer#transformers.TrainingArguments + +## 오류 분석 +- 모델의 손실 기준으로 검증 샘플을 정렬 +- 정방향 패스의 결과와 레이블을 사용하면 손실을 자동 계산 가능 + +## CODE +colab을 사용해 실행 +- colab/MotionClassifier.ipynb + +## Reference +1. 아파치 애로우(Apache Arrow) + - 서로 다른 데이터 인프라가 서로 간의 데이터 공유를 위해 API를 이용할때 발생하는 문제점 중 하나는 직렬화와 역직렬화의 오버헤드가 너무 높다는 것 + - Arrow는 언어, 플랫폼과 상관없이 메모리 상에서 컬럼 구조로 데이터를 정의하여 CPU, GPU에서 메모리를 빠르게 읽고 쓸 수 있도록 해줌 + +
+ +2. 박스 플롯(Box Plot) + - 많은 데이터를 눈으로 확인하기 어려울 때 그림을 이용해 데이터 집합의 범위와 중앙값을 빠르게 확인하는 목적으로 사용 + +
+ +3. WordPiece + - https://wikidocs.net/166826 + +4. Embedding Layer + - 문자 입력에 대해 학습을 요할 때 필요 Layer + - 단어를 의미론적 기하공간에 매핑할 수 있도록 벡터화 시킴 + +5. UMAP + - 차원 축소 기술로 머신러닝 개발자들이 고차원 데이터셋을 이해하고 시각화하기 위해 사용하는 툴 + - umap 동작 원리: https://m.blog.naver.com/myohyun/222421460444 + - umap github: https://github.com/lmcinnes/umap + - UMAP 생성자 파라미터 + +
+ + +6. 오차행렬(confused matrix) + - 학습된 분류 모델이 예측을 수행하며 얼마나 햇갈리고(confused) 있는지도 함께 보여주는 지표 + +
+ + + - True: 예측 클래스 값과 실제 클래스 값이 같음을 의미 + - False: 예측 클래스 값과 실제 클래스 값이 다름을 의미 + - Accuracy = (TN + TP) / (TN + FP + FN + TP) + - 다중 클래스의 경우 자기 클래스는 Positive, 나머지는 모두 Negative로 처리 + +7. f1 점수 + - Precision(정밀도) + - 모델이 True라고 분류한 것 중에 실제 True인 것의 비율 + - Precision = (TP) / (TP + FP) + - Recall(재현율) + - 실제 True인 것중에 모델이 True라고 예측한 것의 비율 + - Recall = (TP) / (TP + FN) + - Ex] 날씨가 맑다/아니다를 예측 + - Precision: 날씨 예측 모델이 맑다로 예측했는데 실제 날씨가 맑았는지를 살펴보는 지표 + - Recall: 실제 날씨가 맑은 날중에서 모델이 맑다고 예측한 비율을 나타내는 지표 + - Precision, Recall은 모두 실제 True인 정답을 모델이 True라고 예측한 경우에 관심이 있으나 보는 관점이 다른 것 + - Accuracy(정확도) + - True를 True로 False를 False라고 예측한 지표(Precision과 Recall은 True만 고려) + - 그러나 Accuracy의 경우 데이터가 한쪽으로 편중될시 문제가 발생 + - 데이터가 한쪽으로 많이 쏠려 있을시 그 Data만 예측해도 정확도가 올라가게 됨 + - F1 Score + - Precision과 Recall의 조화 평균 + - F1 Score = 2 * 1 / ((1/Precision) + (1/Recall)) + - F1 Score는 데이터 Label이 불균형 구조일 때 모델의 성능을 정확히 평가, 성능을 하나의 숫자로 표현이 가능 + + - https://sumniya.tistory.com/26 \ No newline at end of file diff --git a/Transformer/01_TextClassification/colab/MotionClassifier.ipynb b/Transformer/01_TextClassification/colab/MotionClassifier.ipynb new file mode 100644 index 0000000..48760e7 --- /dev/null +++ b/Transformer/01_TextClassification/colab/MotionClassifier.ipynb @@ -0,0 +1 @@ +{"cells":[{"cell_type":"code","source":["!pip3 install torch\n","!pip3 install torchvision"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"POAqZe4M6UNm","executionInfo":{"status":"ok","timestamp":1673341816630,"user_tz":-540,"elapsed":6922,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"9ffa3f0b-6599-4dca-e154-cbfc98daaee1"},"id":"POAqZe4M6UNm","execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (1.13.0+cu116)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch) (4.4.0)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: torchvision in /usr/local/lib/python3.8/dist-packages (0.14.0+cu116)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torchvision) (4.4.0)\n","Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from torchvision) (2.25.1)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from torchvision) (1.21.6)\n","Requirement already satisfied: torch==1.13.0 in /usr/local/lib/python3.8/dist-packages (from torchvision) (1.13.0+cu116)\n","Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.8/dist-packages (from torchvision) (7.1.2)\n","Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision) (4.0.0)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision) (1.24.3)\n"]}]},{"cell_type":"code","source":["!pip install datasets"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"68m-GDr66fxa","executionInfo":{"status":"ok","timestamp":1673341825377,"user_tz":-540,"elapsed":8766,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"fcdb4575-faf7-4211-c4b6-5528526a5df3"},"id":"68m-GDr66fxa","execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting datasets\n"," Downloading datasets-2.8.0-py3-none-any.whl (452 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m452.9/452.9 KB\u001b[0m \u001b[31m23.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.3)\n","Collecting huggingface-hub<1.0.0,>=0.2.0\n"," Downloading huggingface_hub-0.11.1-py3-none-any.whl (182 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m182.4/182.4 KB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting xxhash\n"," Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 KB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting responses<0.19\n"," Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n","Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n","Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n","Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (4.64.1)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from datasets) (1.21.6)\n","Collecting multiprocess\n"," Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 KB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from datasets) (21.3)\n","Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2022.11.0)\n","Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (2.25.1)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (6.0)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.3)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n","Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n","Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (2.1.1)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.4.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.8.2)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->datasets) (3.0.9)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2.10)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n","Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (4.0.0)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n","Collecting urllib3<1.27,>=1.21.1\n"," Downloading urllib3-1.26.13-py2.py3-none-any.whl (140 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 KB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7)\n","Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n","Installing collected packages: xxhash, urllib3, multiprocess, responses, huggingface-hub, datasets\n"," Attempting uninstall: urllib3\n"," Found existing installation: urllib3 1.24.3\n"," Uninstalling urllib3-1.24.3:\n"," Successfully uninstalled urllib3-1.24.3\n","Successfully installed datasets-2.8.0 huggingface-hub-0.11.1 multiprocess-0.70.14 responses-0.18.0 urllib3-1.26.13 xxhash-3.2.0\n"]}]},{"cell_type":"code","execution_count":3,"id":"2f3c6918","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2f3c6918","executionInfo":{"status":"ok","timestamp":1673341831251,"user_tz":-540,"elapsed":5877,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"9827fa97-0000-4b0b-ea1c-245aa2e3b8f6"},"outputs":[{"output_type":"stream","name":"stdout","text":["현재 허브에는 18224개의 데이터셋이 존재\n","처음 10개 데이터셋: ['acronym_identification', 'ade_corpus_v2', 'adversarial_qa', 'aeslc', 'afrikaans_ner_corpus', 'ag_news', 'ai2_arc', 'air_dialogue', 'ajgt_twitter_ar', 'allegro_reviews']\n"]}],"source":["from datasets import list_datasets\n","\n","all_datasets = list_datasets()\n","print(f\"현재 허브에는 {len(all_datasets)}개의 데이터셋이 존재\")\n","print(f\"처음 10개 데이터셋: {all_datasets[:10]}\")"]},{"cell_type":"code","execution_count":4,"id":"feed8efa","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":377,"referenced_widgets":["15be0db60e7447b39d8a3efb0178f99d","6c2a216ef9454384a48332dfcd11e854","321df68850b64edbb4bf68f3efc4666c","2f251ea5146e42c8a10e3e1299d20ee7","e6f31047df2b410d91620c2f04f8714c","48d26eaa385347bb8f1df071dbb3fcf5","37e087967b954cf39bbed8efc00f8910","affcdf3100514e08ae78b45fdff62bb7","d4dbe15a78104e4080f3168d27a28f2f","17d39ba9ceb74e4a971c9d7fe0a388e5","ad9e1452c5a44231a749edefc88ec559","8596765b76d8475687f9a68471baa8c2","245cdb335127490eb54f0b211711187c","ee552a6f31454b0eba764bed5304c447","3ccae33d0f704dd4ab20ed71ad34629d","3e85d7ab6dc9402ea32346d736445024","ca67643c59934657987dbe05ff1f6a79","19ce3569029c443abcd029d7b1e5434f","ec09ef01f31b4c63b70a4f8686084da9","fff4ffa55682415d9f2964ed56978877","67591dd5ed2b4cd4aa72809954922e4e","41a7de832edd405cb11f56b9d7b5ce4b","4d4e5bdc8d534162ba27f70bd5e1f082","16d9f032b1d14503971178f91ccd9957","61e5561b9f374d0baf8d5f73094798fc","1a8d91a9ac674e41a6e0bf62c5b35477","a5addd4c0f114525acde2000dc504386","bbd6396309394aebac8c614a6a93bab6","8da6d9afaae94419b6580759f2760247","c0b1272b05d94b6d803882a8067d6afb","9c4da2a7141f4df88ffc59d3d63c694a","a980d9bda04444ed94d1519d7d628a49","f4a2e61b61d645629758014940686e07","204278e03cfa4697804abf20ff225813","1a82751128d443f4ac5b5303c1b73153","6c8c86abf14d44c5b4efdb368da1a513","bd9de3f5b9574ac3bf3a45d05196e734","e9c04430c58b45518f4e663d4732ebbe","740a2671e40842f79f715ba2d258440b","bc4307eb644e4a6ebfaa308ec79b5911","1f7ab1db7a5c43a386c4097d8469d4a2","e3e4baf588234e2a80af2e8e9858ed5f","31985c31745a4dfebf496417e9b46f3a","d0e5f1132e754fe59287c20f4b94f96b","7c2a5fe9cff34e31ac794deb1e788d8d","acb980cf775e44299b861e54a2eff750","62c0ff2b19c84cdfad5a83ff8720983f","7303d71c059d4066942330e3e70bede0","ac5e2af54b694df5983d1418332e7636","d4a4259ce3c14993af19a12affd34eb8","6f2500756b6f42c391298eeaed5f8854","c98890a867614a26920d4bb067bee304","85754c374f0d43e0bad78444da647c61","fcca2b9f79e0446f9e2f91e0270e9b4b","dd50d5fb200746ccaecb690921ec2c42","7bfe63d5a7c54b4ea7bc6221105d0cbb","e6d52c04d9644613b040bf53957eb46f","97c67c9ea7d9422581abe4a21e091b4b","7fcb248425894c77bef4326f479c1926","36258d1c281a4d2eb26f92513288726d","c0fa09c25d334099899a963c3bc36c00","2da35527796b47eb9ab3cb1cbcaccaf8","3820bb7924a14e27b2b23d83f63d6606","ca8c43945b2a4ec391c00c7b10b48504","1abe53ac1c564bb58c8b7c06ff097d11","b7828189ea0c4d7792da50b67e748efd","bed71be988084af68c0628bcbcfe63f6","17c3146d7a6449f4ad08ea1d3052add1","66ef552e0f0e4591bb39fb6f0fc034d6","5df6a4f46d82455488419dd8e7df3470","07a62cdc075c4e56a4e68189564ac136","c5d0ec37d3cd40558754aa9480eb2a6b","6084c39bce3c4b5ab13d7586c4431a3c","a71d88aedade49038831bc24d2281e3a","562cf26a240949c787ff51dc1f3ef4a2","df95194ca38e43fb83250fa86d096613","bf1dc66bb9164730b4fe171c24604125","c545f017ed8e4a3e8add6a23e7e37314","418167aad1554edea1dc5d1d6b879ccd","812d351083b24a9db1f535b1c6ef5e61","61cfdd6ec9664df98ad6fa2a202318f8","ec5517f772c648f8839125acc7163bd2","bf2332fb0c2c440b9b71f00818936a53","f12f2f547a7e417db26df14ef00c6c80","79be6dfa4f8145d09f072ed6f5cb4d3a","6cfca50926a04bf0b7455de53ed39e07","ede97313536a4a1882e0363a28d771fc","77b2d4741f58419c87725f88409fa34d","774d58af423e4d559b55fb7ca867b73e","2249c0ee4a6c4a12b5c1b004ef0f7bea","8c6a709ec66f4bf89d7dc1d99ebfcd85","b203c5ac560b4c0b91da8d3936c9701e","a56c46fb64984a99acf33eaf1d0749e2","654e5a1bf15944139a8af596fea472ad","57f07b09eaff4e2cbb2baf0f55aa6f56","2f78b83bf6ab4e309657919870f72914","7619571658cd4fa2841a04d78dc1fbe0","cc4c766ab3cd4d31a6a1c2bd62fd1ac1","d546cdab698241b593f2a4668cbbe718","1dcfb16f3b99497fab648fbc508622af","0edf7b00d74e4aa08a2a3dde6f9377d9","68663f8ab1464294ad24fe1550746272","62dec3a2e87744a8accf7e17bdfc4fe1","7e386e08bdd34471b5be7c5b578e3fe8","e0da041ea0a34884a3ac413a0632c05a","dc452bd507d04875922b12d79a12be8c","cdd8485752434ebabbc37d1e4f268e5d","131e10a0bb814925b81350ee521d10ec","9d0a2f1b16554acc8ee8fa798b3d80d5","4e22f63e5af146279cdafa6e14fa0235","3ac8646d66e9466cb19ca475d5c35319","4e20703456eb4983bfd220202193f770","ff1ff119aa864603bc9e179ad039db1c","79c0f1e2dedf459fad9d99c9da23921f","f4536072a0f245ce955bf17c4e69fff7","62e068731b004692b62bfbe3d6a7d4bd","21401b5db07742ccbd904349816513b8","12d18e67c09045aa95cb9e59a9b6e2a8","fca1857b9a8f48d28b74fc06d6986bd5","826dafb23b96401b854502fd7faff689","0f6f60d107854832ab63a53fbfbd079c","d7190fe0ffbe44ed88183863b01ae154","37f0d2cf136240c9a05d09530134731b","4b00f1b6b9a44812ace3d76be6eddc93","43a0ec11689f4195871427b3518a564d","a67d6cc425c94c4a8a114da132392197","46df938913904dff8d9d6ac483529a5c","140e7085dabc494d89eeeeb30d5f8a19","a2f713bb502644fd93663239d7af8c4d","aa0bb486843e4663952242eed3ba1261","1bb5c0a141984ad39f5fb3ce2be79552","f38775a718884ecdbaf317b2369550da"]},"id":"feed8efa","executionInfo":{"status":"ok","timestamp":1673341846622,"user_tz":-540,"elapsed":15373,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"6f88ae9d-a345-410c-e429-2153da87dd9b"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading builder script: 0%| | 0.00/3.97k [00:00\n","
| \n"," | text | \n","label | \n","
|---|---|---|
| 0 | \n","i didnt feel humiliated | \n","0 | \n","
| 1 | \n","i can go from feeling so hopeless to so damned... | \n","0 | \n","
| 2 | \n","im grabbing a minute to post i feel greedy wrong | \n","3 | \n","
| 3 | \n","i am ever feeling nostalgic about the fireplac... | \n","2 | \n","
| 4 | \n","i am feeling grouchy | \n","3 | \n","
| \n"," | text | \n","label | \n","label_name | \n","
|---|---|---|---|
| 0 | \n","i didnt feel humiliated | \n","0 | \n","sadness | \n","
| 1 | \n","i can go from feeling so hopeless to so damned... | \n","0 | \n","sadness | \n","
| 2 | \n","im grabbing a minute to post i feel greedy wrong | \n","3 | \n","anger | \n","
| 3 | \n","i am ever feeling nostalgic about the fireplac... | \n","2 | \n","love | \n","
| 4 | \n","i am feeling grouchy | \n","3 | \n","anger | \n","
\n","파이썬의 str 객체 내부는 사실 배열이므로 문자 수준의 토큰화가 손쉽게 가능"]},{"cell_type":"code","execution_count":17,"id":"9b5c18a4","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9b5c18a4","executionInfo":{"status":"ok","timestamp":1673341847310,"user_tz":-540,"elapsed":12,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"81675c55-2892-4542-fd6d-d6fbf120999b"},"outputs":[{"output_type":"stream","name":"stdout","text":["['T', 'o', 'k', 'e', 'n', 'i', 'z', 'i', 'n', 'g', ' ', 't', 'e', 'x', 't', ' ', 'i', 's', ' ', 'a', ' ', 'c', 'o', 'r', 'e', ' ', 't', 'a', 's', 'k', ' ', 'o', 'f', ' ', 'N', 'L', 'P', '.']\n"]}],"source":["text = \"Tokenizing text is a core task of NLP.\"\n","tokenized_text = list(text)\n","print(tokenized_text)"]},{"cell_type":"code","execution_count":18,"id":"acf0acdd","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"acf0acdd","executionInfo":{"status":"ok","timestamp":1673341847310,"user_tz":-540,"elapsed":10,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"88f785d3-4cc2-4635-c044-e7368fb02582"},"outputs":[{"output_type":"stream","name":"stdout","text":["{' ': 0, '.': 1, 'L': 2, 'N': 3, 'P': 4, 'T': 5, 'a': 6, 'c': 7, 'e': 8, 'f': 9, 'g': 10, 'i': 11, 'k': 12, 'n': 13, 'o': 14, 'r': 15, 's': 16, 't': 17, 'x': 18, 'z': 19}\n"]}],"source":["token2idx = {ch: idx for idx, ch in enumerate(sorted(set(tokenized_text)))}\n","print(token2idx)"]},{"cell_type":"code","execution_count":19,"id":"ff118c22","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ff118c22","executionInfo":{"status":"ok","timestamp":1673341847310,"user_tz":-540,"elapsed":9,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"d86f3c49-412b-43fa-e063-431158532361"},"outputs":[{"output_type":"stream","name":"stdout","text":["[5, 14, 12, 8, 13, 11, 19, 11, 13, 10, 0, 17, 8, 18, 17, 0, 11, 16, 0, 6, 0, 7, 14, 15, 8, 0, 17, 6, 16, 12, 0, 14, 9, 0, 3, 2, 4, 1]\n"]}],"source":["input_ids = [token2idx[token] for token in tokenized_text]\n","print(input_ids)"]},{"cell_type":"code","execution_count":20,"id":"8508c09d","metadata":{"id":"8508c09d","executionInfo":{"status":"ok","timestamp":1673341851106,"user_tz":-540,"elapsed":3803,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"outputs":[],"source":["import torch\n","import torch.nn.functional as F"]},{"cell_type":"code","execution_count":21,"id":"4ce2e4e1","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4ce2e4e1","executionInfo":{"status":"ok","timestamp":1673341851106,"user_tz":-540,"elapsed":31,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"f93c10f1-a432-4141-c108-e40acb976646"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["torch.Size([38, 20])"]},"metadata":{},"execution_count":21}],"source":["input_ids = torch.tensor(input_ids)\n","one_hot_encodings = F.one_hot(input_ids, num_classes=len(token2idx))\n","one_hot_encodings.shape"]},{"cell_type":"markdown","id":"8b8449c2","metadata":{"id":"8b8449c2"},"source":["38개의 입력 토큰 각각에 20차원의 원-핫 벡터가 만들어짐\n","\n","5 .....len(20)
\n","14.....len(20)
\n","12.....len(20)
\n",".\n",".\n",". .....len(20)
\n","2 .....len(20)
\n","4 .....len(20)
\n","1 .....len(20)
\n","\n","각 토큰별 20개의 원소를 가진 벡터가 만들어짐"]},{"cell_type":"code","execution_count":22,"id":"58874e4b","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"58874e4b","executionInfo":{"status":"ok","timestamp":1673341851107,"user_tz":-540,"elapsed":28,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"e5ff92cb-cc63-455c-f447-8ed4e1b8846c"},"outputs":[{"output_type":"stream","name":"stdout","text":["토큰: T\n","텐서 인덱스: 5\n","원-핫 인코딩: tensor([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n"]}],"source":["print(f\"토큰: {tokenized_text[0]}\")\n","print(f\"텐서 인덱스: {input_ids[0]}\")\n","print(f\"원-핫 인코딩: {one_hot_encodings[0]}\")"]},{"cell_type":"markdown","id":"5729671a","metadata":{"id":"5729671a"},"source":["## 단어 토큰화\n","텍스트를 문자가 아닌 단어로 분할하고 각 단어를 정수로 매핑"]},{"cell_type":"code","execution_count":23,"id":"6fbe209f","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6fbe209f","executionInfo":{"status":"ok","timestamp":1673341851107,"user_tz":-540,"elapsed":23,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"d3cd0fbe-5a53-411d-f084-ac04b70efff5"},"outputs":[{"output_type":"stream","name":"stdout","text":["['Tokenizing', 'text', 'is', 'a', 'core', 'task', 'of', 'NLP.']\n"]}],"source":["tokenized_text = text.split()\n","print(tokenized_text)"]},{"cell_type":"markdown","id":"f8988f71","metadata":{"id":"f8988f71"},"source":["단어 토큰화는 단어에 곡용, 활용형, 철자 오류가 포함되어 어휘 사전이 금세 수백만 개까지 늘어나게 됨"]},{"cell_type":"markdown","id":"bfb042f0","metadata":{"id":"bfb042f0"},"source":["단어를 토큰화 하게 되면 어휘사전이 커지게 되고 이러면 신경망의 파라미터 역시 많이 필요하게 됨\n","
\n","- 어휘 사전의 크기를 제한하는 일반적인 방법은 드물게 등장하는 단어는 무시하는 것. 이러한 단어는 'UNK' 토큰으로 매핑\n","- 하지만 이렇게 되면 토큰화 과정에서 중요 정보 일부를 잃게 됨\n","- 모든 입력 정보와 일부 입력 구조를 유지하는 문자 토큰화와 단어 토큰화를 절충하는 방법으로 **부분단어 토큰화(subword tokenization)** 라는 방법"]},{"cell_type":"markdown","id":"95a96577","metadata":{"id":"95a96577"},"source":["## 부분단어 토큰화\n","- 부분단어 토큰화는 문자 단어 토큰화 + 단어 토큰화 의 장점을 결합\n","- 드물게 등장하는 단어를 더 작은 단위로 나누면 모델이 복잡한 단어나 철자 오류를 처리하기 용이\n","- 다른 방법으론 입력 길이를 적절한 크기로 유지하기 위해 자주 등장하는 단어를 고유한 항목으로 유지\n","
\n","- NLP 분야에서 널리 사용되는 부분단어 토큰화 중 먼저 BERT와 DistilBERT의 토크나이저 **WordPiece**"]},{"cell_type":"code","source":["!pip install transformers"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NmnMUChO63ZZ","executionInfo":{"status":"ok","timestamp":1673341860819,"user_tz":-540,"elapsed":9733,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"c0b8dfaa-3526-445d-db30-cdf01b04fed8"},"id":"NmnMUChO63ZZ","execution_count":24,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting transformers\n"," Downloading transformers-4.25.1-py3-none-any.whl (5.8 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.8/5.8 MB\u001b[0m \u001b[31m79.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n","Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n"," Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m69.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.11.1)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.8.2)\n","Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (21.3)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.4.0)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.26.13)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n","Installing collected packages: tokenizers, transformers\n","Successfully installed tokenizers-0.13.2 transformers-4.25.1\n"]}]},{"cell_type":"code","execution_count":25,"id":"2379be10","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":145,"referenced_widgets":["2cbf56d11bd84bd39a2e8f00be50ab98","fa29a3affda84dc38e53a7ec02e4a1f2","1886290ec37f4f7eb7dced6d494cbe6a","b37bdca77d064cf2b7ef2bf21edc554b","14709700a96448cda2f036f9886463f9","e9230e8b794f433e9ed48f62a244c1f3","16ab6bdfe98a44dcb15fb374ffdd7dcb","f9ac65f7c12b4c70b1481fd0052ebcab","a49d46601b5d4395bd61aa89b3c4f261","fa363e3b514f47909eb1caf3bda7d4ac","4c2ddc339f2948cda9331332583f71ef","815b635b8e2d429ab30cc56170508f2b","2783f2b3b469440e829ecf06ce441e24","740089b7671147a8b2aa5334dbe299a1","a55dbf76cedf4c51a80127d1368f00fa","fcc0299f56d54e069e252951755dd8d9","5a16a657767448af9953262cbe566fda","279728f2f8e842af9b078cdd58d71af0","a3452d2825e040f9aad5667274d1d9d9","c8b548fcf7e14265ade8eb891b4717bf","3ef5ff5cb4c54628bb3bc720a65f93a2","eb8260186835427ab020dbdf4d484794","5f67cfa6f99c4045a77ef8f221ed8a72","b07045b7f68944aeb2c8bd0e41994b93","137fcc68aefc48179ababf622aad0776","96b98cd1ec9946d58e510739cf43324e","c6eb6b1626314623b55ad235895da2f9","72cad00b563945b6b5c9c43f8eb34172","2971a235130f4f72b0b8486364d63ad5","c70eb58ccb2347d38c8cde483c47191f","b7b7225c7d7943e280b421c760b72caa","147c8d4a28d146a792e7c8a037bcb011","823d188f499e47f6a8d87503c4ad77f7","029fdd7a3b0842229d7b0a93c5867fa1","438bcf79e8694464ad3a846a0643be03","a8baed453dd9425fb3d788033508119d","375bd083c65b4bf882f21c29c858890c","149d349aa3cb46c588d164bd37b84117","a12bce3f9ff841f683b19a6b581c587f","4d864c90cd41415e9e610edf9fba526b","323fcf7620f04d9da487b8628e229e5b","c8259c53289d4100a9fab111d220bea3","7ac6ebbf3dc040ae9a9b3614557da598","6d19cf0811e64cd7b29cb63458060fad"]},"id":"2379be10","executionInfo":{"status":"ok","timestamp":1673341874956,"user_tz":-540,"elapsed":14138,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"8f74a0e1-dd3f-43e8-f284-26c904b9be1d"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading: 0%| | 0.00/28.0 [00:00\n","- padding=True: 배치에 있는 가장 긴 샘플 크기에 맞춰 샘플을 0으로 패딩\n","- truncation=True: 모델의 최대 문맥 크기에 맞춰 샘플을 잘라냄"]},{"cell_type":"code","execution_count":33,"id":"9658e034","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9658e034","executionInfo":{"status":"ok","timestamp":1673341874958,"user_tz":-540,"elapsed":8,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"edf35851-8dfa-49ae-e48a-4c7e85ecabef"},"outputs":[{"output_type":"stream","name":"stdout","text":["{'input_ids': [[101, 1045, 2134, 2102, 2514, 26608, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1045, 2064, 2175, 2013, 3110, 2061, 20625, 2000, 2061, 9636, 17772, 2074, 2013, 2108, 2105, 2619, 2040, 14977, 1998, 2003, 8300, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"]}],"source":["print(tokenize(emotions[\"train\"][:2]))"]},{"cell_type":"code","execution_count":34,"id":"03458041","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":113,"referenced_widgets":["69c863e980624e10bb6afad72fed6cd1","3c160b0fb3d7466aa0f79c6233d16001","8b26dd18f285460296a17d6b856f30aa","d795813bd1e447ec8ca874a4eb58c806","8b9b4713c05e402491fa537414d0bd0b","d49fc731f3634038b245ed8b19fec175","ac1f10abc80449b3b570f462332de626","834b4dee42f84269801adc0f5517a89f","6c363fc2f9bd454f86bd477dada30a71","17d0256a8547455fbe4e4f6935cea7a4","f622e7b698bf47bcb1233230e6771772","048922deeb3141b1805c22f41a880ecc","6148410294ec48c3804ed15e7034cb4f","70773097ffad4c9b85b6a739a514e1b3","3da5ccb880f848dcaef28bb694c4bb70","c9dcf555150b4ecd80addf1c1bb33824","5ab0a87049344e46929a316bc2a2e3e9","deb4b8225c6d4fdba6f2efbb33b17000","a7953cfec05a4585b366d3d8e4b2123b","b912f40f4cc247f3a782da9507ef3e09","1a2e990827f248e7bc47da5ef7f17826","31014192818d46ff8b3671e3c5d180de","0177ae63703f4b01889c07ad4cf86e29","02a9d73f776e431198ace480d231d10b","3036423ebe6a49ad91f0de6e728d0ad8","85c41307b3c84b5a86a24f8a2463a939","9a097fa5b27749c7aa379a4ca349bf5b","e21e33b65f604a56b7ee6c3839362574","41bd67ce77564fa5be2527f800620262","48ba903b498a4dbe82b77dd7df3b28c5","4d0d3d035fc6444fa43b487ce81bb981","5ad971ef99af4d379967cf291d13ab74","a8c5dfa23d0f46d5a2b03350abb62c07"]},"id":"03458041","executionInfo":{"status":"ok","timestamp":1673341875660,"user_tz":-540,"elapsed":709,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"f24f9a52-ed17-4b4d-e482-e2a83330c933"},"outputs":[{"output_type":"display_data","data":{"text/plain":[" 0%| | 0/1 [00:00\n","\n","- batched=True: 트윗을 배치로 인코딩\n","- batch_size=None: 전체 데이터셋이 하나의 배치로 tokenize() 함수에 적용, 입력 텐서와 어텐션 마스크는 전역적으로 동일한 크기로 생성"]},{"cell_type":"code","execution_count":35,"id":"3de1ab5c","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3de1ab5c","executionInfo":{"status":"ok","timestamp":1673341875660,"user_tz":-540,"elapsed":4,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"b30e5c75-b3d2-4e6a-e93c-35a19694756f"},"outputs":[{"output_type":"stream","name":"stdout","text":["['text', 'label', 'input_ids', 'attention_mask']\n"]}],"source":["print(emotions_encoded[\"train\"].column_names)"]},{"cell_type":"markdown","id":"c3d9224c","metadata":{"id":"c3d9224c"},"source":["# 텍스트 분류 모델 훈련\n","\n","### 특성 추출\n","사전 훈련된 모델을 수정하지 않고 은닉 상태를 특성(feature)으로 사용해 분류 모델을 훈련\n","\n","### 미세 튜닝\n","사전 훈련된 모델의 파라미터도 업데이트하기 위해 전체 모델을 엔드-투-엔드로 훈련"]},{"cell_type":"markdown","id":"2b923107","metadata":{"id":"2b923107"},"source":["## 1. 텍스트 분류 모델 훈련 - 특성 추출 방법\n"]},{"cell_type":"markdown","id":"c6560f7f","metadata":{"id":"c6560f7f"},"source":["사전 훈련된 모델 사용하기"]},{"cell_type":"code","execution_count":36,"id":"e6f48da5","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":121,"referenced_widgets":["6d20ccf8cbfc434d83d29cf94d66fa2f","563f6e63d6334ffdb92b824ca4543e26","3a3663e4ccf447178e720efe74226eb2","bd5032bb6bd04505a16d6aaeb344a411","843844a0068c4252931450f24da6b86e","bcd73e1f781c4d1d950ae79b4f6787bf","3490bc468f2e4bbeb57e76aaf9706308","2b8f7b15d44346e8a86b7414be26c8f3","e3bad1e7fa8142ce9d618d0c8b1a2225","402cabec74604cc09d0fff59dd529621","4882a7357dd3468491f3bf78eec1909c"]},"id":"e6f48da5","executionInfo":{"status":"ok","timestamp":1673341889298,"user_tz":-540,"elapsed":13640,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"495b6d23-a7ac-4d6a-931f-95091aa040ab"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading: 0%| | 0.00/268M [00:00\n","-> 6개의 입력 토큰마다 768차원의 벡터가 반환"]},{"cell_type":"code","execution_count":42,"id":"9a5b950a","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9a5b950a","executionInfo":{"status":"ok","timestamp":1673341892897,"user_tz":-540,"elapsed":9,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"f0fc13d5-071e-429f-ee2b-2513a406595e"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["torch.Size([1, 768])"]},"metadata":{},"execution_count":42}],"source":["outputs.last_hidden_state[:,0].size()"]},{"cell_type":"code","execution_count":43,"id":"c6eb5006","metadata":{"id":"c6eb5006","executionInfo":{"status":"ok","timestamp":1673341892897,"user_tz":-540,"elapsed":4,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"outputs":[],"source":["def extract_hidden_states(batch):\n"," inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}\n"," \n"," # 마지막 은닉 상태를 추출\n"," with torch.no_grad():\n"," last_hidden_state = model(**inputs).last_hidden_state\n"," \n"," # [CLS] 토큰에 대한 벡터를 반환\n"," return {\"hidden_state\": last_hidden_state[:,0].cpu().numpy()}"]},{"cell_type":"code","execution_count":44,"id":"1faae4d9","metadata":{"id":"1faae4d9","executionInfo":{"status":"ok","timestamp":1673341892897,"user_tz":-540,"elapsed":4,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"outputs":[],"source":["emotions_encoded.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])"]},{"cell_type":"code","execution_count":45,"id":"49cf0fca","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":113,"referenced_widgets":["101a4116f12b44f6b426fc2814944eb8","9edbf9a8ba5d4142a0a12e5ea696da5c","8af71848005147cfaedd0c9ec9d4fd49","44dbd17197c64b538c63a610f7f31617","c7fbfa047d884faeb5969ae896515fc1","84928d15dbe2497baa439cb5721b83a1","ba72138f0e604758b4490c754875af78","b0e164b8f84946f88721e0c0069120a1","6e54fca8c14542cabb8b8a4b33e2dbf8","0bf2839bc8d64d2e94ad85881c74bf92","6ef1bc813fcd44e88a28492317397f9a","ec33154d5df8415098870a8cfa336916","b1611173b99e4e7e887289ec11b22549","b2b36893a8404cbeb820029a11f026c1","1026b11d38b64ec88111620d4e6496b5","a40d5fc95a36471d90292263ecbf3a9a","1f92f00548c1492bb9d5fe9a827c1465","b8fc14d3ad2e493cab95946f99eaa22d","8674806cef2647afa397d1a1a67aeb9c","1afb862c628b422788c23230a7c33263","456d3d915eab4f77b0bf0f2e0326d818","92f5f4755e724e12a59fe33e372e84e4","2fd74fded781498f86d9b097b7bebd2d","b3f0a364d8db417e95971d9bc93cd894","ba119c289d864b8ab108e9cf52a49820","01aa5eadc6a946528e32244d30383ed7","c51fa6e6e9864a7bb0be5bfb9f71c637","1fd5b81af51d4b30b17f630586339560","f0cc4fc15d62434499f1e0fefe2f03b3","3d10c62f029449bcbc574169ac502748","ecb19ac1b1664734a661aa34c037e9b8","73c4003f63bc4da1bffd26ba93218857","e869dea153bf4883ba3e883359e4ba99"]},"id":"49cf0fca","executionInfo":{"status":"ok","timestamp":1673341936842,"user_tz":-540,"elapsed":43949,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"ad796a8e-2d7c-4be6-8df2-03eae34e0955"},"outputs":[{"output_type":"display_data","data":{"text/plain":[" 0%| | 0/16 [00:00\n","은닉 상태를 입력 특성으로 사용하고 레이블을 타깃으로 사용"],"metadata":{"id":"pIOakwuJ9K3v"},"id":"pIOakwuJ9K3v"},{"cell_type":"code","source":["import numpy as np\n","\n","X_train = np.array(emotions_hidden[\"train\"][\"hidden_state\"])\n","X_valid = np.array(emotions_hidden[\"validation\"][\"hidden_state\"])\n","Y_train = np.array(emotions_hidden[\"train\"][\"label\"])\n","Y_valid = np.array(emotions_hidden[\"validation\"][\"label\"])\n","\n","X_train.shape, X_valid.shape"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fmVDUYhH9gWa","executionInfo":{"status":"ok","timestamp":1673342197640,"user_tz":-540,"elapsed":7,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"e2af6fb0-be6c-4965-f794-201abf2e5bc7"},"id":"fmVDUYhH9gWa","execution_count":47,"outputs":[{"output_type":"execute_result","data":{"text/plain":["((16000, 768), (2000, 768))"]},"metadata":{},"execution_count":47}]},{"cell_type":"markdown","source":["## 훈련 세트 시각화\n","은닉 상태로 모델을 훈련하기 전에 분류하려는 감정에 대한 유용한 표현을 제공하는지 확인이 필요
\n","\n","768차원의 은닉 상태를 시각화하기 어렵기에 UMAP 알고리즘을 사용해 이 벡터를 2D로 투영
\n","UMAP은 특성이 [0,1] 범위에 놓일 때 잘 작동. 사이킷런의 MinMaxScaler를 적용한 후 umap-learn 라이브러리의 UMAP 구현으로 은닉 상태의 차원을 축소"],"metadata":{"id":"dWkueZ3m-FFb"},"id":"dWkueZ3m-FFb"},{"cell_type":"code","source":["!pip uninstall umap"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bYDxENRtADzq","executionInfo":{"status":"ok","timestamp":1673342785584,"user_tz":-540,"elapsed":12206,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"4f259645-7939-440c-9ab1-efcc711e55c3"},"id":"bYDxENRtADzq","execution_count":53,"outputs":[{"output_type":"stream","name":"stdout","text":["Found existing installation: umap 0.1.1\n","Uninstalling umap-0.1.1:\n"," Would remove:\n"," /usr/local/lib/python3.8/dist-packages/umap-0.1.1.dist-info/*\n"," /usr/local/lib/python3.8/dist-packages/umap/*\n"," Would not remove (might be manually added):\n"," /usr/local/lib/python3.8/dist-packages/umap/aligned_umap.py\n"," /usr/local/lib/python3.8/dist-packages/umap/distances.py\n"," /usr/local/lib/python3.8/dist-packages/umap/layouts.py\n"," /usr/local/lib/python3.8/dist-packages/umap/parametric_umap.py\n"," /usr/local/lib/python3.8/dist-packages/umap/plot.py\n"," /usr/local/lib/python3.8/dist-packages/umap/sparse.py\n"," /usr/local/lib/python3.8/dist-packages/umap/spectral.py\n"," /usr/local/lib/python3.8/dist-packages/umap/umap_.py\n"," /usr/local/lib/python3.8/dist-packages/umap/utils.py\n"," /usr/local/lib/python3.8/dist-packages/umap/validation.py\n","Proceed (Y/n)? Y\n"," Successfully uninstalled umap-0.1.1\n"]}]},{"cell_type":"code","source":["!pip install umap-learn"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xybNOFKY_mcB","executionInfo":{"status":"ok","timestamp":1673342794193,"user_tz":-540,"elapsed":4302,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"8ea84a9f-5555-4bf6-f408-d6c810321e0f"},"id":"xybNOFKY_mcB","execution_count":54,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: umap-learn in /usr/local/lib/python3.8/dist-packages (0.5.3)\n","Requirement already satisfied: scikit-learn>=0.22 in /usr/local/lib/python3.8/dist-packages (from umap-learn) (1.0.2)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from umap-learn) (1.21.6)\n","Requirement already satisfied: numba>=0.49 in /usr/local/lib/python3.8/dist-packages (from umap-learn) (0.56.4)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from umap-learn) (4.64.1)\n","Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.8/dist-packages (from umap-learn) (1.7.3)\n","Requirement already satisfied: pynndescent>=0.5 in /usr/local/lib/python3.8/dist-packages (from umap-learn) (0.5.8)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from numba>=0.49->umap-learn) (57.4.0)\n","Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.8/dist-packages (from numba>=0.49->umap-learn) (5.2.0)\n","Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.8/dist-packages (from numba>=0.49->umap-learn) (0.39.1)\n","Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from pynndescent>=0.5->umap-learn) (1.2.0)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.22->umap-learn) (3.1.0)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata->numba>=0.49->umap-learn) (3.11.0)\n"]}]},{"cell_type":"code","source":["# from umap import UMAP\n","import umap.umap_ as umap\n","from sklearn.preprocessing import MinMaxScaler\n","\n","# 특성 스케일을 [0,1] 범위로 조정\n","X_scaled = MinMaxScaler().fit_transform(X_train)\n","# UMAP 객체를 생성하고 훈련\n","mapper = umap.UMAP(n_components=2, metric=\"cosine\").fit(X_scaled)\n","\n","# 2D 임베딩의 데이터프레임을 생성\n","df_emb = pd.DataFrame(mapper.embedding_, columns=[\"X\", \"Y\"])\n","df_emb[\"label\"] = Y_train\n","df_emb.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"Sx0BapUg-xoq","executionInfo":{"status":"ok","timestamp":1673343223945,"user_tz":-540,"elapsed":42777,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"56c1711b-2f35-4d77-9bb2-51bc2fa92f77"},"id":"Sx0BapUg-xoq","execution_count":61,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" X Y label\n","0 4.256140 7.099303 0\n","1 -2.679181 6.564047 0\n","2 5.314543 3.535637 3\n","3 -2.102647 4.541106 2\n","4 -3.089052 4.619032 3"],"text/html":["\n","
| \n"," | X | \n","Y | \n","label | \n","
|---|---|---|---|
| 0 | \n","4.256140 | \n","7.099303 | \n","0 | \n","
| 1 | \n","-2.679181 | \n","6.564047 | \n","0 | \n","
| 2 | \n","5.314543 | \n","3.535637 | \n","3 | \n","
| 3 | \n","-2.102647 | \n","4.541106 | \n","2 | \n","
| 4 | \n","-3.089052 | \n","4.619032 | \n","3 | \n","
\n","이 은닉 상태를 사용해 사이킷런의 로지스틱 회귀(logistic regression) 모델을 훈련"],"metadata":{"id":"G9UX5LkODxbZ"},"id":"G9UX5LkODxbZ"},{"cell_type":"code","source":["from sklearn.linear_model import LogisticRegression\n","\n","# 수렴을 보장하기 위해 'max_iter'를 증가\n","lr_clf = LogisticRegression(max_iter=3000)\n","lr_clf.fit(X_train, Y_train)\n","lr_clf.score(X_valid, Y_valid)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7wRPfAEDD8Kn","executionInfo":{"status":"ok","timestamp":1673344183711,"user_tz":-540,"elapsed":224801,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"18062b77-087e-4fce-d6c0-95e1c0b063dd"},"id":"7wRPfAEDD8Kn","execution_count":64,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.633"]},"metadata":{},"execution_count":64}]},{"cell_type":"markdown","source":["오차 행렬(confusion matrix)\n","\n","
\n","오차 행렬은 진짜 레이블과 예측 레이블의 관계를 보여줌"],"metadata":{"id":"Bm3KZUPZFtj5"},"id":"Bm3KZUPZFtj5"},{"cell_type":"code","source":["from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix\n","\n","def plot_confusion_matrix(y_preds, y_true, labels):\n"," cm = confusion_matrix(y_true, y_preds, normalize=\"true\")\n"," fig, ax = plt.subplots(figsize=(6,6))\n"," disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\n"," disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax, colorbar=False)\n"," plt.title(\"Normalized confusion matrix\")\n"," plt.show()\n","\n","y_preds = lr_clf.predict(X_valid)\n","plot_confusion_matrix(y_preds, Y_valid, labels)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":404},"id":"8ykaFc2WFq_3","executionInfo":{"status":"ok","timestamp":1673344451707,"user_tz":-540,"elapsed":650,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"cfa77d1c-4e0f-4902-b5a8-3e60a17e3460"},"id":"8ykaFc2WFq_3","execution_count":65,"outputs":[{"output_type":"display_data","data":{"text/plain":["
\n","미세 튜닝 방식에서는 은닉 상태를 고정된 특성으로 사용하지 않고 전체 모델을 훈련 (분류 헤드는 미분 가능 해야 함)\n"],"metadata":{"id":"5KZ4dG9dHIVB"},"id":"5KZ4dG9dHIVB"},{"cell_type":"markdown","source":["분류 모델에 입력으로 사용하는 은닉 상태를 훈련하면 분류 작업에 적합하지 않은 데이터를 다룬다는 문제를 회피 할 수 있음"],"metadata":{"id":"0wmNDecZHroJ"},"id":"0wmNDecZHroJ"},{"cell_type":"markdown","source":["### 사전 훈련된 모델 로드\n","\n","특성 기반 방식에서 사용한 것과 같은 사전 훈련된 DistilBERT 모델을 사용
\n","\n","But, AutoModel 대신 AutoModelForSequenceClassification을 사용, 이 모델은 사전 훈련된 모델 출력위에 베이스 모델과 함께 쉽게 훈련할 수 있는 분류 헤드가 있음
\n","\n","분류 헤드의 출력 크기를 설정하기 위해 모델이 예측할 레이블 개수(여기서는 6)을 지정"],"metadata":{"id":"N642fZjSH0YQ"},"id":"N642fZjSH0YQ"},{"cell_type":"code","source":["from transformers import AutoModelForSequenceClassification\n","\n","num_labels = 6\n","model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to(device))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1qo-aIBbG9jc","executionInfo":{"status":"ok","timestamp":1673345010467,"user_tz":-540,"elapsed":3708,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"abbffc8d-f3b7-4f04-9e27-bbb9d1c1358d"},"id":"1qo-aIBbG9jc","execution_count":66,"outputs":[{"output_type":"stream","name":"stderr","text":["Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_projector.bias', 'vocab_transform.weight']\n","- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'pre_classifier.weight', 'classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]}]},{"cell_type":"markdown","source":["모델 일부가 랜덤하게 초기화된다는 경고를 보냄. (분류 헤드가 아직 훈련되지 않아 발생)"],"metadata":{"id":"R_Ik1FnzIy1c"},"id":"R_Ik1FnzIy1c"},{"cell_type":"markdown","source":["#### 성공 지표 정의\n","\n","미세 튜닝 과정에서 모델 성능을 평가할 때 사용할 측정 지표를 정의
\n","훈련하는 동안 성능을 모니터링하기 위해 Trainer에 사용할 compute_metrics() 함수를 정의"],"metadata":{"id":"YQ1rmWVjI5p4"},"id":"YQ1rmWVjI5p4"},{"cell_type":"code","source":["from sklearn.metrics import accuracy_score, f1_score\n","\n","def compute_metrics(pred):\n"," labels = pred.label_ids\n"," preds = pred.predictions.argmax(-1)\n"," f1 = f1_score(labels, preds, average=\"weighted\")\n"," acc = accuracy_score(labels, preds)\n"," return {\"accuracy\":acc, \"f1\":f1}"],"metadata":{"id":"lYfMGf2gJQNJ","executionInfo":{"status":"ok","timestamp":1673345269650,"user_tz":-540,"elapsed":923,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"id":"lYfMGf2gJQNJ","execution_count":67,"outputs":[]},{"cell_type":"code","source":["from transformers import Trainer, TrainingArguments\n","\n","batch_size=64\n","logging_steps = len(emotions_encoded[\"train\"]) // batch_size\n","model_name = f\"{model_ckpt}-finetuned-emotion\"\n","training_args = TrainingArguments(output_dir=model_name,\n"," num_train_epochs=2,\n"," learning_rate=2e-5,\n"," per_device_train_batch_size=batch_size,\n"," per_device_eval_batch_size=batch_size,\n"," weight_decay=0.01,\n"," evaluation_strategy=\"epoch\",\n"," disable_tqdm=False,\n"," logging_steps=logging_steps,\n"," push_to_hub=False,\n"," save_strategy=\"epoch\",\n"," load_best_model_at_end=True,\n"," log_level=\"error\")"],"metadata":{"id":"TzQOKwCXJ9M0","executionInfo":{"status":"ok","timestamp":1673345650008,"user_tz":-540,"elapsed":459,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"id":"TzQOKwCXJ9M0","execution_count":69,"outputs":[]},{"cell_type":"code","source":["from transformers import Trainer\n","\n","trainer = Trainer(model=model, args=training_args, compute_metrics=compute_metrics, train_dataset=emotions_encoded[\"train\"], eval_dataset=emotions_encoded[\"validation\"], tokenizer=tokenizer)\n","trainer.train()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":228},"id":"zX24iuGHLF0u","executionInfo":{"status":"ok","timestamp":1673345986689,"user_tz":-540,"elapsed":237842,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"d77ec0a6-0847-4e4b-bef8-41044907f25d"},"id":"zX24iuGHLF0u","execution_count":70,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.8/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n"," warnings.warn(\n"]},{"output_type":"display_data","data":{"text/plain":["
| Epoch | \n","Training Loss | \n","Validation Loss | \n","Accuracy | \n","F1 | \n","
|---|---|---|---|---|
| 1 | \n","0.828300 | \n","0.308017 | \n","0.911000 | \n","0.908939 | \n","
| 2 | \n","0.244800 | \n","0.213043 | \n","0.921000 | \n","0.920982 | \n","
"]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=500, training_loss=0.5365411682128907, metrics={'train_runtime': 237.4156, 'train_samples_per_second': 134.785, 'train_steps_per_second': 2.106, 'total_flos': 720342861696000.0, 'train_loss': 0.5365411682128907, 'epoch': 2.0})"]},"metadata":{},"execution_count":70}]},{"cell_type":"code","source":["preds_output = trainer.predict(emotions_encoded[\"validation\"])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"A58G56dvMZX3","executionInfo":{"status":"ok","timestamp":1673346023963,"user_tz":-540,"elapsed":6167,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"fee6f7b4-4b89-48f1-fc63-aaa6cea1ac73"},"id":"A58G56dvMZX3","execution_count":71,"outputs":[{"output_type":"display_data","data":{"text/plain":[" 1 rows × 25 columns 2 rows × 25 columns 4 rows × 25 columns "]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=1575, training_loss=0.1560986812531002, metrics={'train_runtime': 513.6597, 'train_samples_per_second': 73.473, 'train_steps_per_second': 3.066, 'total_flos': 863012377186080.0, 'train_loss': 0.1560986812531002, 'epoch': 3.0})"]},"metadata":{},"execution_count":40}]},{"cell_type":"code","source":["text_de = \"Jeff Dean ist ein Informatiker bei Google in Kalifornien\"\n","tag_text(text_de, tags, trainer.model, xlmr_tokenizer)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"id":"KhAcv-0Tg570","executionInfo":{"status":"ok","timestamp":1675821640752,"user_tz":-540,"elapsed":27,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"a3967115-e433-4c8b-a517-e363936451c8"},"execution_count":41,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" 0 1 2 3 4 5 6 7 8 9 \\\n","Tokens "]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[" "]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[" "]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[" "]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[" "]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[" "]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=2145, training_loss=0.17714588014118043, metrics={'train_runtime': 694.0678, 'train_samples_per_second': 74.171, 'train_steps_per_second': 3.09, 'total_flos': 1140291491923584.0, 'train_loss': 0.17714588014118043, 'epoch': 3.0})"]},"metadata":{},"execution_count":69}]},{"cell_type":"markdown","source":["각 언어의 테스트 세트에서 모델의 성능 확인"],"metadata":{"id":"37XkUPr59WEv"}},{"cell_type":"code","source":["for lang in langs:\n"," f1 = evaluate_lang_performance(lang, trainer)\n"," print(f\"[{lang}] 데이터셋에서 [de-fr] 모델의 f1 점수: {f1:.3f}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":359,"referenced_widgets":["40ee5833271c494cb982535613847865","9cd43cf140984ac594dc4449ea856e1a","d2656be4605a408b8960d620155d6872","40c9c94449694fbbaf08b0a4da3a2470","5fa3437f023c44d3887654afac5164f6","5ee625774f254d7bbfdb58fc2b88c032","6e949eae282e4fcfb12a5fa8f678b3f8","aca5de1935654297aa171b16d4ae2117","8ada8e3b825043589a33076d479fa97a","33cc02879cb148b998f9da0fff984862","b1a1ce3dadb3403bb8c20f2ca16e3513","323822f38a9e48a0b69b9e72713ad9fd","5b58cd19a040458785de40b350d0a0a2","afb03e48c9a64e459182c7969105d693","0f3f7f44d66a40ccb0c4c0d409aceff1","78d9f1251e3f43368ef4400057df0e9c","f475fece0cc94fc3875d60e697b712f9","be5245964f2f496792d8513dbd681e42","2d428917487c4f18abc84819b6ddbcde","6a24b1cebe05436da45963e95550b673","d14809e23834488fb4f2a385f0bb03c1","c992e5f51eb94d079c00d433f819606f","d6c483f1f2a74e0b88a864e9524a52f6","4894b9cf2ee5495bafa58b8e78af5497","90c9e7c500534b758f21b5a9567252c6","1dfee2d2e19c40e0b5606e01a4f1b1ac","5207663a59f642c9935f742990337395","521599d1ca3047fbb6736a55a84386a7","345d1df57cd347979e56b8434bbe9579","700d93168336461a9b5c221f06b778a4","a0e796962708476a8d43898e8f9e7eb4","b54b5dd342c147a98f0c859f3e80d807","ea15d48dae7d4f3f810df52a2c445966"]},"id":"epQuCOif9Yk1","executionInfo":{"status":"ok","timestamp":1675825989266,"user_tz":-540,"elapsed":29774,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"bc69b4b0-bf53-4cb0-c82a-6c49455d40c2"},"execution_count":70,"outputs":[{"output_type":"display_data","data":{"text/plain":[" 0%| | 0/13 [00:00"],"text/html":[]},"metadata":{}},{"output_type":"stream","name":"stderr","text":["WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/29f5d57a48779f37ccb75cb8708d1095448aad0713b425bdc1ff9a4a128a56e4/cache-1dce373a1b796a7e.arrow\n","WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/29f5d57a48779f37ccb75cb8708d1095448aad0713b425bdc1ff9a4a128a56e4/cache-6c6f6ff4a59e7dd1.arrow\n","WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/29f5d57a48779f37ccb75cb8708d1095448aad0713b425bdc1ff9a4a128a56e4/cache-bd9403b1b214a85b.arrow\n"]},{"output_type":"stream","name":"stdout","text":["[de] 데이터셋에서 [de-fr] 모델의 f1 점수: 0.869\n"]},{"output_type":"display_data","data":{"text/plain":["
\n","정방향 패스의 결과와 레이블을 사용하면 손실은 자동으로 계산 가능."],"metadata":{"id":"HD_r8G7GNKu8"},"id":"HD_r8G7GNKu8"},{"cell_type":"code","source":["# 손실과 예측 레이블을 반환\n","from torch.nn.functional import cross_entropy\n","\n","def forward_pass_with_label(batch):\n"," # 모든 입력 텐서를 모델과 같은 장치로 이동\n"," inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}\n","\n"," with torch.no_grad():\n"," output = model(**inputs)\n"," pred_label = torch.argmax(output.logits, axis=-1)\n"," loss = cross_entropy(output.logits, batch[\"label\"].to(device), reduction=\"none\")\n","\n"," # 다른 데이터셋 열과 호환되도력 출력을 CPU로 옮김\n"," return {\"loss\":loss.cpu().numpy(), \"predicted_label\":pred_label.cpu().numpy()}"],"metadata":{"id":"_vElx2WqNeIE","executionInfo":{"status":"ok","timestamp":1673346476379,"user_tz":-540,"elapsed":4,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"id":"_vElx2WqNeIE","execution_count":76,"outputs":[]},{"cell_type":"markdown","source":["map 메서드로 이 함수를 적용해 모든 샘플의 손실을 구함"],"metadata":{"id":"R1pbWEwvOQls"},"id":"R1pbWEwvOQls"},{"cell_type":"code","source":["# 데이터셋을 다시 파이토치 텐서로 변환\n","emotions_encoded.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])\n","\n","# 손실 값을 계산\n","emotions_encoded[\"validation\"] = emotions_encoded[\"validation\"].map(forward_pass_with_label, batched=True, batch_size=16)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["80aa883b3cf649ceb6e5b85ed5ea5509","313593925c2a4141a64864e53e122063","0b9065b551c8412f80340a506e6c685b","2c451fd1646d400d8d24adef4c5edaab","44040ea5691b408a8f206b0acc3b9ea7","838fadbc513045ce9492e00a834433af","aeb48bd095004b65b3eccd405c997500","d94d1a4d44ad4a4594f1008be77cfe51","0bdcc5fd663c45d79e31442b9abb40e5","8e0cd84d88f0431eb265ecf5114b2288","7a4511d0bb9143569319cf5ae8c7836f"]},"id":"9BUPB6pkOPJh","executionInfo":{"status":"ok","timestamp":1673346600282,"user_tz":-540,"elapsed":13943,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"4d3981b7-b2b5-44ed-be79-fabb1adf30f7"},"id":"9BUPB6pkOPJh","execution_count":77,"outputs":[{"output_type":"display_data","data":{"text/plain":[" 0%| | 0/125 [00:00 데이터에 레이블을 부여하는 프로세스는 모두 완벽하지 않음
\n","잘못 레이블링된 샘플이 존재 할 수 있음 앞의 방식을 사용하면 이런 레이블을 빠르게 찾아 수정이 가능"],"metadata":{"id":"KWR1b4ztPZhW"},"id":"KWR1b4ztPZhW"},{"cell_type":"markdown","source":["#### 데이터셋의 특이사항\n","텍스트 데이터에서는 입력에 포함된 특수 문자나 문자열이 모델 예측에 큰 영향을 미치기도 함
\n","모델의 가장 나쁜 예측을 들여다보게 되면 이런 특성을 찾게 되고, 데이터를 정제하거나 비슷한 샘플을 추가하면 모델이 안정됨"],"metadata":{"id":"hHxQvP99P4g5"},"id":"hHxQvP99P4g5"},{"cell_type":"code","source":["df_test.sort_values(\"loss\", ascending=False).head(10)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":363},"id":"YBLpveUHQDXr","executionInfo":{"status":"ok","timestamp":1673346975822,"user_tz":-540,"elapsed":1126,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"f57ccba6-0b46-4b98-83c5-aed7b0c160a6"},"id":"YBLpveUHQDXr","execution_count":79,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" text label \\\n","1963 i called myself pro life and voted for perry w... joy \n","1950 i as representative of everything thats wrong ... surprise \n","1274 i am going to several holiday parties and i ca... joy \n","1111 im lazy my characters fall into categories of ... joy \n","1509 i guess this is a memoir so it feels like that... joy \n","1870 i guess i feel betrayed because i admired him ... joy \n","1801 i feel that he was being overshadowed by the s... love \n","318 i felt ashamed of these feelings and was scare... fear \n","1500 i guess we would naturally feel a sense of lon... anger \n","882 i feel badly about reneging on my commitment t... love \n","\n"," predicted_label loss \n","1963 sadness 5.367970 \n","1950 sadness 5.284013 \n","1274 sadness 5.276999 \n","1111 fear 5.139698 \n","1509 fear 5.066939 \n","1870 sadness 5.032856 \n","1801 sadness 4.932096 \n","318 sadness 4.787760 \n","1500 sadness 4.679534 \n","882 sadness 4.383337 "],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," text \n"," label \n"," predicted_label \n"," loss \n"," \n"," \n"," 1963 \n"," i called myself pro life and voted for perry w... \n"," joy \n"," sadness \n"," 5.367970 \n"," \n"," \n"," 1950 \n"," i as representative of everything thats wrong ... \n"," surprise \n"," sadness \n"," 5.284013 \n"," \n"," \n"," 1274 \n"," i am going to several holiday parties and i ca... \n"," joy \n"," sadness \n"," 5.276999 \n"," \n"," \n"," 1111 \n"," im lazy my characters fall into categories of ... \n"," joy \n"," fear \n"," 5.139698 \n"," \n"," \n"," 1509 \n"," i guess this is a memoir so it feels like that... \n"," joy \n"," fear \n"," 5.066939 \n"," \n"," \n"," 1870 \n"," i guess i feel betrayed because i admired him ... \n"," joy \n"," sadness \n"," 5.032856 \n"," \n"," \n"," 1801 \n"," i feel that he was being overshadowed by the s... \n"," love \n"," sadness \n"," 4.932096 \n"," \n"," \n"," 318 \n"," i felt ashamed of these feelings and was scare... \n"," fear \n"," sadness \n"," 4.787760 \n"," \n"," \n"," 1500 \n"," i guess we would naturally feel a sense of lon... \n"," anger \n"," sadness \n"," 4.679534 \n"," \n"," \n"," \n","882 \n"," i feel badly about reneging on my commitment t... \n"," love \n"," sadness \n"," 4.383337 \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," text \n"," label \n"," predicted_label \n"," loss \n"," \n"," \n"," 19 \n"," i had lunch with an old friend and it was nice... \n"," joy \n"," joy \n"," 0.015154 \n"," \n"," \n"," 205 \n"," i cannot wait for school to end so i can chang... \n"," joy \n"," joy \n"," 0.015296 \n"," \n"," \n"," 669 \n"," i am not feeling very joyful today its been a ... \n"," joy \n"," joy \n"," 0.015381 \n"," \n"," \n"," 604 \n"," i don t like to use the h word recklessly but ... \n"," joy \n"," joy \n"," 0.015426 \n"," \n"," \n"," 11 \n"," i was dribbling on mums coffee table looking o... \n"," joy \n"," joy \n"," 0.015474 \n"," \n"," \n"," 1513 \n"," i have also been getting back into my gym rout... \n"," joy \n"," joy \n"," 0.015520 \n"," \n"," \n"," 578 \n"," i got to christmas feeling positive about the ... \n"," joy \n"," joy \n"," 0.015627 \n"," \n"," \n"," 1320 \n"," im feeling positive but its impossible to desc... \n"," joy \n"," joy \n"," 0.015640 \n"," \n"," \n"," 329 \n"," i have had my treasury selection on the front ... \n"," joy \n"," joy \n"," 0.015814 \n"," \n"," \n"," \n","1263 \n"," i feel this way about blake lively \n"," joy \n"," joy \n"," 0.015964 \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," de \n"," fr \n"," it \n"," en \n"," \n"," \n"," \n","Number of training examples \n"," 12580 \n"," 4580 \n"," 1680 \n"," 1180 \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," 10 \n"," 11 \n"," \n"," \n"," Tokens \n"," 2.000 \n"," Einwohnern \n"," an \n"," der \n"," Danziger \n"," Bucht \n"," in \n"," der \n"," polnischen \n"," Woiwodschaft \n"," Pommern \n"," . \n"," \n"," \n"," \n","Tags \n"," O \n"," O \n"," O \n"," O \n"," B-LOC \n"," I-LOC \n"," O \n"," O \n"," B-LOC \n"," B-LOC \n"," I-LOC \n"," O \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," LOC \n"," ORG \n"," PER \n"," \n"," \n"," train \n"," 6186 \n"," 5366 \n"," 5810 \n"," \n"," \n"," validation \n"," 3172 \n"," 2683 \n"," 2893 \n"," \n"," \n"," \n","test \n"," 3180 \n"," 2573 \n"," 3071 \n","
\n","PER, LOC, ORG 빈도 분포가 대체로 각 분할에서 동일. 따라서 검증 세트와 테스트 세트는 NER 태그의 일반화 능력을 평가하는데 적절"],"metadata":{"id":"UGMvN5mAB1lU"}},{"cell_type":"markdown","source":["# 다중 언어 트랜스포머\n","다중 언어 트랜스포머의 훈련 과정과 아키텍처는 단일 언어 트랜스포머와 비슷, 다만 사전 훈련에 사용하는 말뭉치가 여러 언어의 문서로 구성될 뿐\n","
\n","다중 언어 트랜스포머 모델은 일반적으로 3가지 방식으로 평가\n","
\n","## 1. en\n","- 영어 훈련 데이터에서 미세 튜닝한 다음 각 언어의 테스트 세트에서 평가\n","\n","## 2. each\n","- 언어별 성능을 측정하기 위해 단일 언어의 테스트 세트에서 미세 튜닝하고 평가\n","\n","## 3. all\n","- 모든 훈련 데이터에서 미세 튜닝해 각 언어의 테스트 세트에서 평가\n","\n"],"metadata":{"id":"wi71g2zCCX_p"}},{"cell_type":"markdown","source":["# XLM-R 토큰화\n","XLM-R은 WordPiece 토크나이저 대신 100개 언어의 텍스트에서 훈련된 SentencePiece라는 토크나이저를 사용"],"metadata":{"id":"4FANse1eD9u2"}},{"cell_type":"code","source":["!pip install transformers"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"enKCHp9yFGm6","executionInfo":{"status":"ok","timestamp":1675821054744,"user_tz":-540,"elapsed":9035,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"9f9f1135-0bfd-45d1-bc4e-09e39cb2daee"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting transformers\n"," Downloading transformers-4.26.0-py3-none-any.whl (6.3 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m49.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.12.0)\n","Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (23.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n","Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n"," Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m102.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n","Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.26.14)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n","Installing collected packages: tokenizers, transformers\n","Successfully installed tokenizers-0.13.2 transformers-4.26.0\n"]}]},{"cell_type":"code","source":["from transformers import AutoTokenizer\n","\n","bert_model_name = \"bert-base-cased\"\n","xlmr_model_name = \"xlm-roberta-base\"\n","bert_tokenizer = AutoTokenizer.from_pretrained(bert_model_name)\n","xlmr_tokenizer = AutoTokenizer.from_pretrained(xlmr_model_name)"],"metadata":{"id":"5OnUUkFDERfK","executionInfo":{"status":"ok","timestamp":1675821078805,"user_tz":-540,"elapsed":24081,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"colab":{"base_uri":"https://localhost:8080/","height":241,"referenced_widgets":["f1f69f016e5f4d179fc1a3a3639e194f","4f5da1ad913e472c9d77e1cdc4f8e2b6","ebd7b36421f948439548e7903ed81ee6","96f5ba8063074c748b250f3d38de52f2","31ddd298f1894a069583d6d93d2e45eb","2ead4e84970946d9859ce37da2175d3e","acd513b021d24cd4b127755190f90302","016e44c2b25545e693c1a316ad17deaf","4185518d566c42468315e518bf9fcfe7","0ed70f7d08fa49819bb9c60eafaec297","0644205b912848a7a7f77830b7258223","94f51cdae0494f0dbcf0b341047c90dc","efa3383f5b524d56ae509f4d8e278d22","fb036d925be044438f01290f948f8e02","8365cc260cba472d85e6a02ded253ca1","0e2d034462e94a178aa4cfa54343bc69","8c91896dddd142b1996ab976b14cc1c7","d582d329ca4449349a6e08e11ac7d25c","c71c37c8c3f443f88c38d94ba7a597c7","00e0165acad84074a1b02f60b8993d8c","345a8b6eefe348de982f2fe100e4b94c","57553ee721924d69b2846ab8a05e4305","4604e0047d924a839e969b1f219a8d5e","c24a2e3f6fa842b6ac52996933480550","5ee0e7b975c74d4cbdc765e7d82e2ef0","de8687af54af49b99a3f05ec8654c83d","89139f8f31c846c48a3d0b329b881bde","33cd59f8716d4f7ea40821c7774a38fa","2e793a8f212a427186a427b54c00fe33","6a6f5218d1c54be7a30777ceb4fd6b00","85f8bbb7d8b24b5a95c7fb674d6cb43a","60207316f72e435f9266225c5f6fbc19","ff8103828112408cbf16c5c6d13b2a33","f753628d026a471a8b6f7987011dfe91","670b05b90bdf4587beb6a9e1b3f998e8","8b91d4f194844d68bd81acd3438b7ba2","07e98f59b8fb4ffe9c21f62f2e1b3ae5","6ccf052943204fb7be8c2a2987237148","d94df671dfa14c02bac4f12819c0748e","d9978751d44d486b8d99c812088d29ce","c9dfffd412d045a283e46566158deee7","69e2116a570c4b61b55b7dd7ff0545e7","b991d0341ba748a5a86e36682c350a10","098f8beefedf487e8e620a33e5b4873d","5bfc16d5cd14405d969e20261f913e40","2ddc82fde5284c5fa22e80a1ded0c4de","0090c677db7045df9b5ce4935468b238","9f9f9d032e9c468990f61a9769620a6d","332ea6b461d744bdaf2edbf3048d80ea","fdcfc65469e04a4baef6def0234b3884","021e8ac000564e9692aa09f083f36704","dab4bb5c9bd44bff9e91421be0977990","6d7f64577b9c48a8848a84dfded91fe4","9d602ef4136d43f0a5b6beee8555a3bb","bc060a9491da4500b6279c7e4f8880e3","80d6515bf0e145c189303a631f4ea08f","801affdca836427988a60f8d21deda58","f83ba0cbd9cc4a1588c7e76b43f068c0","e1b3263abf434c8d94b02b0be45f9434","d97430ac276e4589b503b90886cb8a00","df18f7c7b94a40ae828ea9eac0a60b34","925a4454e6e449218c42585198563cbf","55df9e5858dd4284b40482ebfa346142","e90cebb0049341fdb70f11727cbf5e79","1d630e8957ad45728b9a539d32692cc1","f4ca2927907243ffa08b2baa44941069","d1fd8e9e28984643b4f55abd35f77840","123eb17fe6c243688872a572c2cc88ec","8cac781872fb407facc613e411f8d0c4","fc3d29bb3e2d42a696a2b51aa9e19459","784055e398204ce8a9029852e73ef5bd","bab93cded8a045f299b7374306048456","914f3a746bd547f9a951132454e7818a","c8d7f27575f1400a932be83e3b2c2e07","e4b4b462d7af44efa4be4c8e1259da99","a4017fc65bb64bb391f7f90fcad7629c","686bc45dd1384f158656a37caa5e4563"]},"outputId":"c51b37aa-7370-4d7b-a9c0-e093e4b9c23b"},"execution_count":14,"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading (…)okenizer_config.json: 0%| | 0.00/29.0 [00:00', '▁Jack', '▁Spar', 'row', '▁love', 's', '▁Net', '▁York', '!', '']\n"]}]},{"cell_type":"markdown","source":["## 토큰화 파이프라인\n","토큰화를 다룰 때 문자열을 모델에 주입할 정수로 변환하는 연산으로만 여겼지만 이는 전적으로 옳은 것은 아님. 실제 처리 파이프 라인은 아래와 같이 4 단계로 구성됨\n","\n","1. 정규화\n","2. 사전 토큰화\n","3. 토크나이저 모델\n","4. 사후처리\n","\n","1. 정규화\n","- 원시 문자열을 깨끗하게 만들기 위해 적용하는 일련의 연산, 대게 이런 연산에는 공백과 악센트가 붙는 문자를 제거하는 작업 등이 포함\n","- 유니코드 정규화(https://unicode.org/reports/tr15/)는 많은 토크나이저에서 적용하는 또 다른 일반적 정규화 연산\n","\n","2. 사전 토큰화(pretokenization)\n","- 텍스트를 더 작은 객체로 분할하며 훈련 마지막에 생성되는 토큰의 상한선을 제공\n","- 사전 토큰화가 텍스트를 단어로 분할하고 최종 토큰은 이 단어의 일부가 된다 생각하면 됨\n","- 영어, 독어 등 많은 유럽어족의 언어에서 문자열이 일반적으로 공백과 구두점을 기준 삼아 단어로 분할\n"," - 이런 단어는 파이프라인의 다음 단계에서 BPE, 유니그램 알고리즘을 사용해 부분단어(subword)로 분할하기 더 쉬움\n","\n","3. 토크나이저 모델\n","- 부분단어 분할 모델을 단어에 적용, 이 모델은 단어를 부분단어로 나눠 어휘사전의 크기와 OOV 토큰의 개수를 줄이는 역할을 함\n","- 부분단어 알고리즘\n"," - BPE,\n"," - 유니그램\n"," - WordPiece\n","\n","4. 사후처리\n","- 토큰 리스트에 부가적인 변환을 적용\n"," - Ex] 입력 토큰 인덱스의 시퀀스 처음과 끝에 특수 토큰을 추가하는 경우 등"],"metadata":{"id":"JIsrxh8oF-Q7"}},{"cell_type":"markdown","source":["# SentencePiece 토크나이저\n","유니그램이라는 부분단어 분할 방식을 기반으로 각 입력 텍스트를 유니코드 문자 시퀀스로 인코딩\n","- 일본어 등의 많은 언어에 공백문자가 있지 않는데 SentencePiece의 또 다른 고유한 특징은 공백문자가 유니코드 기호 U+2581 또는 '_' 문자에 할당 되므로 언어별 사전 토크나이저에 의존하지 않고 정확하게 시퀀스 복원이 가능 "],"metadata":{"id":"kFQNqaQgJLdW"}},{"cell_type":"markdown","source":["# 개체명 인식을 위한 트랜스포머\n","\n","\n"],"metadata":{"id":"tfce7a6ELLGY"}},{"cell_type":"markdown","source":["토큰 분류 작업에서 부분단어 처리 방법\n","
\n","예를 들어 이름 Christa는 Chr와 #ista로 토큰화 됨 이중 어느 단어에 (아니면 두 단어 모두에) B-PER 레이블을 할당해야 할까요?\n","
\n","BERT 논문에서 저자들은 첫번째 부분단어('Chr')에 할당하고 이어지는 부분단어는 무시, 무시한 부분단어를 IGN으로 표시. 나중에 후처리 단계에서 첫 번째 부분단어의 예측 레이블을 후속 부분단어로 쉽게 전파 가능"],"metadata":{"id":"rYnY6i3ANrS2"}},{"cell_type":"markdown","source":["# 트랜스포머 모델 클래스\n","트랜스포머스는 아키텍처와 작업마다 전용 클래스를 제공\n","
\n","작업에 연관된 모델 클래스 이름은
\n","트랜스포머스는 기존 모델을 특정 작업에 맞춰 쉽게 확장 가능하도록 설계됨. 사전 훈련된 모델에서 가중치를 로드하고 작업에 특화된 헬퍼 함수를 사용. 특정 용도의 사용자 정의 모델을 만들 수 있음"],"metadata":{"id":"SNgK8H2mQFzH"}},{"cell_type":"markdown","source":["## 바디와 헤드\n","트랜스포머스의 모델들은 바디(body)와 헤드(head)로 나뉜 모델 구조를 가짐\n","
\n","모델의 마지막 층이 후속 작업에 맞는 층으로 변환. 이 마지막 층이 모델의 **헤드** 이며 작업에 특화\n","
\n","헤드를 제외한 모델의 나머지 부분을 **바디**라 함. 작업에 특화되지 않은 토큰 임베딩과 트랜스포머 층이 포함. 모델의 바디는 BertModel 또는 GPT2Model 같은 클래스로 구현되며 마지막 바디의 마지막 층의 은닉 상태를 반환\n",""],"metadata":{"id":"ZLy-BBzWQeEg"}},{"cell_type":"markdown","source":["## 토큰 분류를 위한 사용자 정의 모델 만들기"],"metadata":{"id":"2cSDj1_vTZMP"}},{"cell_type":"code","source":["import torch.nn as nn\n","from transformers import XLMRobertaConfig\n","from transformers.modeling_outputs import TokenClassifierOutput\n","from transformers.models.roberta.modeling_roberta import RobertaModel\n","from transformers.models.roberta.modeling_roberta import RobertaPreTrainedModel\n","\n","class XLMRobertaForTokenClassification(RobertaPreTrainedModel):\n"," config_class = XLMRobertaConfig\n","\n"," def __init__(self, config):\n"," super().__init__(config)\n"," self.num_labels = config.num_labels\n"," # 모델 바디를 로드\n"," self.roberta = RobertaModel(config, add_pooling_layer=False)\n"," # 토큰 분류 헤드를 준비\n"," self.dropout = nn.Dropout(config.hidden_dropout_prob)\n"," self.classifier = nn.Linear(config.hidden_size, config.num_labels)\n"," # 가중치를 로드하고 초기화\n"," self.init_weights()\n","\n"," def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n"," # 모델 바디를 사용해 인코더 표현을 얻음\n"," outputs = self.roberta(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, **kwargs)\n"," # 인코더 표현을 헤드에 통과\n"," sequence_output = self.dropout(outputs[0])\n"," logits = self.classifier(sequence_output)\n"," # 손실을 계산\n"," loss = None\n"," if labels is not None:\n"," loss_fct = nn.CrossEntropyLoss()\n"," loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n"," # 모델 출력 객체를 반환\n"," return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions)"],"metadata":{"id":"dUoKNNdjTWk3","executionInfo":{"status":"ok","timestamp":1675821080583,"user_tz":-540,"elapsed":1798,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":17,"outputs":[]},{"cell_type":"markdown","source":["- config 클래스는 새로운 모델을 초기화할 때 표준 XLM-R 설정을 사용하도록 도와줌\n","- super() 메서드로 RobertaPreTrainedModel 클래스의 초기화 함수를 호출\n"," - 사전 훈련된 가중치의 초기화나 로딩을 처리\n","- add_pooling_layer=False로 지정하여 [CLS] 토큰에 해당하는 은닉 상태 외에 모든 은닉 상태가 반환되도록 함\n"," - add_pooling_layer의 기본값은 True로 첫 번째 토큰의 은닉 상태만 밀집 층과 활성화 함수에 통과시켜 출력\n","- RobertaPreTrainedModel 클래스에서 상속된 init_weights() 메서드를 호출해 가중치를 초기화\n"," - 모델 바디에 사전훈련된 가중치가 로드되고 토큰 분류 헤드의 가중치가 랜덤하게 초기화 될 것"],"metadata":{"id":"9CHPVLJsjJp6"}},{"cell_type":"code","source":["# [tags를 정의했던 위 코드]\n","# tags = panx_ch[\"de\"][\"train\"].features[\"ner_tags\"].feature\n","# ClassLabel(names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None)\n","\n","index2tag = {idx: tag for idx, tag in enumerate(tags.names)}\n","tag2index = {tag: idx for idx, tag in enumerate(tags.names)}"],"metadata":{"id":"Bic1OxxBkJGU","executionInfo":{"status":"ok","timestamp":1675821080583,"user_tz":-540,"elapsed":5,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["from transformers import AutoConfig\n","\n","xlmr_config = AutoConfig.from_pretrained(xlmr_model_name, num_labels=tags.num_classes, id2label=index2tag, label2id=tag2index)"],"metadata":{"id":"kU7yY0YGkWAX","executionInfo":{"status":"ok","timestamp":1675821081789,"user_tz":-540,"elapsed":1210,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":19,"outputs":[]},{"cell_type":"markdown","source":["AutoConfig 클래스는 모델 구조의 청사진을 가짐. AutoModel.from_pretrained(model_ckpt)로 모델을 로드할 때 모델에 연관된 설정 파일이 자동으로 다운로드 됨. but, 클래스 개수나 레이블 이름 등을 수정하고자 한다면 커스터마이징 하려는 매개변수로 이 설정 파일을 먼저 로드해야 함"],"metadata":{"id":"oH2i5fvHleg-"}},{"cell_type":"code","source":["# from_pretrained() 함수를 사용해 모델 가중치를 로드\n","import torch\n","\n","device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","xlmr_model = (XLMRobertaForTokenClassification.from_pretrained(xlmr_model_name, config=xlmr_config).to(device))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":156,"referenced_widgets":["ee35beb51277432a80c6c507ea941e81","9977e174493d4522b4ad8643c0a7b534","1ba89974c87e4c6a818f6a85c03b55c8","96861e3ec0124949a8dfbbd9a3f8ae4d","52f2ce5e3e4246b8a503b21ad531e9ed","5d26e3b64c4444f3a772c33f95906e7e","42c224b56c9c4b548abd21c97122ad41","e8c85c704de8418eb3363b8ad9a6146c","e8eb5682d2334e1d8833a209f2dbd826","464ca3b6b1694d93becbabba2bc5f2e6","11c5f0454f774718a6d4efb4ba63cae7"]},"id":"g5F0vtmhl9hX","executionInfo":{"status":"ok","timestamp":1675821105942,"user_tz":-540,"elapsed":23379,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"cb75f3fd-ab52-4ab9-b170-291b0cfe1aa3"},"execution_count":20,"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading (…)\"pytorch_model.bin\";: 0%| | 0.00/1.12G [00:00 ▁Jack ▁Spar row ▁love s ▁Net ▁York ! \n","Input IDs 0 21763 37456 15555 5161 7 10086 5753 38 2"],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," \n"," \n"," Tokens \n"," <s> \n"," ▁Jack \n"," ▁Spar \n"," row \n"," ▁love \n"," s \n"," ▁Net \n"," ▁York \n"," ! \n"," </s> \n"," \n"," \n"," \n","Input IDs \n"," 0 \n"," 21763 \n"," 37456 \n"," 15555 \n"," 5161 \n"," 7 \n"," 10086 \n"," 5753 \n"," 38 \n"," 2 \n"," ▁Jack ▁Spar row ▁love s ▁Net ▁York ! \n","Tags B-LOC B-PER B-PER B-PER B-LOC B-PER B-ORG B-ORG B-PER B-LOC"],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," \n"," \n"," Tokens \n"," <s> \n"," ▁Jack \n"," ▁Spar \n"," row \n"," ▁love \n"," s \n"," ▁Net \n"," ▁York \n"," ! \n"," </s> \n"," \n"," \n"," \n","Tags \n"," B-LOC \n"," B-PER \n"," B-PER \n"," B-PER \n"," B-LOC \n"," B-PER \n"," B-ORG \n"," B-ORG \n"," B-PER \n"," B-LOC \n"," ▁2.000 ▁Einwohner n ▁an ▁der ▁Dan zi ger ▁Buch ... ▁Wo \n","\n"," 16 17 18 19 20 21 22 23 24 \n","Tokens i wod schaft ▁Po mmer n ▁ . \n","\n","[1 rows x 25 columns]"],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," ... \n"," 15 \n"," 16 \n"," 17 \n"," 18 \n"," 19 \n"," 20 \n"," 21 \n"," 22 \n"," 23 \n"," 24 \n"," \n"," \n"," \n","Tokens \n"," <s> \n"," ▁2.000 \n"," ▁Einwohner \n"," n \n"," ▁an \n"," ▁der \n"," ▁Dan \n"," zi \n"," ger \n"," ▁Buch \n"," ... \n"," ▁Wo \n"," i \n"," wod \n"," schaft \n"," ▁Po \n"," mmer \n"," n \n"," ▁ \n"," . \n"," </s> \n","
\n","이 작업에 도움이되는 **word_idx()** 함수가 존재"],"metadata":{"id":"tlkzUfpsuhnQ"}},{"cell_type":"code","source":["word_ids = tokenized_input.word_ids()\n","pd.DataFrame([tokens, word_ids], index=[\"Tokens\", \"Word IDs\"])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":141},"id":"qRM2SXAAu8L-","executionInfo":{"status":"ok","timestamp":1675821109157,"user_tz":-540,"elapsed":27,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"fa1f0f28-c7dc-4ed9-bbbb-53a9ac316bae"},"execution_count":27,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" 0 1 2 3 4 5 6 7 8 9 ... \\\n","Tokens ▁2.000 ▁Einwohner n ▁an ▁der ▁Dan zi ger ▁Buch ... \n","Word IDs None 0 1 1 2 3 4 4 4 5 ... \n","\n"," 15 16 17 18 19 20 21 22 23 24 \n","Tokens ▁Wo i wod schaft ▁Po mmer n ▁ . \n","Word IDs 9 9 9 9 10 10 10 11 11 None \n","\n","[2 rows x 25 columns]"],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," ... \n"," 15 \n"," 16 \n"," 17 \n"," 18 \n"," 19 \n"," 20 \n"," 21 \n"," 22 \n"," 23 \n"," 24 \n"," \n"," \n"," Tokens \n"," <s> \n"," ▁2.000 \n"," ▁Einwohner \n"," n \n"," ▁an \n"," ▁der \n"," ▁Dan \n"," zi \n"," ger \n"," ▁Buch \n"," ... \n"," ▁Wo \n"," i \n"," wod \n"," schaft \n"," ▁Po \n"," mmer \n"," n \n"," ▁ \n"," . \n"," </s> \n"," \n"," \n"," \n","Word IDs \n"," None \n"," 0 \n"," 1 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 4 \n"," 4 \n"," 5 \n"," ... \n"," 9 \n"," 9 \n"," 9 \n"," 9 \n"," 10 \n"," 10 \n"," 10 \n"," 11 \n"," 11 \n"," None \n"," 와 같은 특수 토큰과 부분단어의 레이블을 -100으로 설정해 훈련하는 동안 마스킹 처리"],"metadata":{"id":"PJLzO46q1xyj"}},{"cell_type":"code","source":["previous_word_idx = None\n","label_ids = []\n","\n","for word_idx in word_ids:\n"," if word_idx is None or word_idx == previous_word_idx:\n"," label_ids.append(-100)\n"," elif word_idx != previous_word_idx:\n"," label_ids.append(labels[word_idx])\n"," previous_word_idx = word_idx\n","\n","labels = [index2tag[l] if l != -100 else \"IGN\" for l in label_ids]\n","index = [\"Tokens\", \"Word IDs\", \"Label IDs\", \"Labels\"]\n","\n","pd.DataFrame([tokens, word_ids, label_ids, labels], index=index)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"7-zfZERZ1wV3","executionInfo":{"status":"ok","timestamp":1675821109157,"user_tz":-540,"elapsed":26,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"1eddd254-07af-4424-ff1c-1d5a19654005"},"execution_count":28,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" 0 1 2 3 4 5 6 7 8 \\\n","Tokens ▁2.000 ▁Einwohner n ▁an ▁der ▁Dan zi ger \n","Word IDs None 0 1 1 2 3 4 4 4 \n","Label IDs -100 0 0 -100 0 0 5 -100 -100 \n","Labels IGN O O IGN O O B-LOC IGN IGN \n","\n"," 9 ... 15 16 17 18 19 20 21 22 23 \\\n","Tokens ▁Buch ... ▁Wo i wod schaft ▁Po mmer n ▁ . \n","Word IDs 5 ... 9 9 9 9 10 10 10 11 11 \n","Label IDs 6 ... 5 -100 -100 -100 6 -100 -100 0 -100 \n","Labels I-LOC ... B-LOC IGN IGN IGN I-LOC IGN IGN O IGN \n","\n"," 24 \n","Tokens \n","Word IDs None \n","Label IDs -100 \n","Labels IGN \n","\n","[4 rows x 25 columns]"],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," ... \n"," 15 \n"," 16 \n"," 17 \n"," 18 \n"," 19 \n"," 20 \n"," 21 \n"," 22 \n"," 23 \n"," 24 \n"," \n"," \n"," Tokens \n"," <s> \n"," ▁2.000 \n"," ▁Einwohner \n"," n \n"," ▁an \n"," ▁der \n"," ▁Dan \n"," zi \n"," ger \n"," ▁Buch \n"," ... \n"," ▁Wo \n"," i \n"," wod \n"," schaft \n"," ▁Po \n"," mmer \n"," n \n"," ▁ \n"," . \n"," </s> \n"," \n"," \n"," Word IDs \n"," None \n"," 0 \n"," 1 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 4 \n"," 4 \n"," 5 \n"," ... \n"," 9 \n"," 9 \n"," 9 \n"," 9 \n"," 10 \n"," 10 \n"," 10 \n"," 11 \n"," 11 \n"," None \n"," \n"," \n"," Label IDs \n"," -100 \n"," 0 \n"," 0 \n"," -100 \n"," 0 \n"," 0 \n"," 5 \n"," -100 \n"," -100 \n"," 6 \n"," ... \n"," 5 \n"," -100 \n"," -100 \n"," -100 \n"," 6 \n"," -100 \n"," -100 \n"," 0 \n"," -100 \n"," -100 \n"," \n"," \n"," \n","Labels \n"," IGN \n"," O \n"," O \n"," IGN \n"," O \n"," O \n"," B-LOC \n"," IGN \n"," IGN \n"," I-LOC \n"," ... \n"," B-LOC \n"," IGN \n"," IGN \n"," IGN \n"," I-LOC \n"," IGN \n"," IGN \n"," O \n"," IGN \n"," IGN \n","
\n","seqeval 라이브러리를 사용"],"metadata":{"id":"G19YlRxrX2do"}},{"cell_type":"code","source":["!pip install seqeval"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fPg8R9AaYzo9","executionInfo":{"status":"ok","timestamp":1675821119967,"user_tz":-540,"elapsed":8465,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"374f35ae-a34d-4e8f-d33c-8449201699fb"},"execution_count":32,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting seqeval\n"," Downloading seqeval-1.2.2.tar.gz (43 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 KB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.8/dist-packages (from seqeval) (1.21.6)\n","Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.8/dist-packages (from seqeval) (1.0.2)\n","Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.7.3)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.1.0)\n","Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.2.0)\n","Building wheels for collected packages: seqeval\n"," Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16179 sha256=c763e2631784352be3bf6f1d8e2dc9e8819e6ff8c1582528886bfa3a8a170802\n"," Stored in directory: /root/.cache/pip/wheels/ad/5c/ba/05fa33fa5855777b7d686e843ec07452f22a66a138e290e732\n","Successfully built seqeval\n","Installing collected packages: seqeval\n","Successfully installed seqeval-1.2.2\n"]}]},{"cell_type":"code","source":["from seqeval.metrics import classification_report\n","\n","y_true = [[\"O\", \"O\", \"O\", \"B-MISC\", \"I-MISC\", \"I-MISC\", \"O\"], [\"B-PER\", \"I-PER\", \"O\"]]\n","y_pred = [[\"O\", \"O\", \"B-MISC\", \"I-MISC\", \"I-MISC\", \"I-MISC\", \"O\"], [\"B-PER\", \"I-PER\", \"O\"]]\n","\n","print(classification_report(y_true, y_pred))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Os-Bhu0EYL-8","executionInfo":{"status":"ok","timestamp":1675821120368,"user_tz":-540,"elapsed":406,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"179b1fe3-9138-4606-ffda-79454d01bf50"},"execution_count":33,"outputs":[{"output_type":"stream","name":"stdout","text":[" precision recall f1-score support\n","\n"," MISC 0.00 0.00 0.00 1\n"," PER 1.00 1.00 1.00 1\n","\n"," micro avg 0.50 0.50 0.50 2\n"," macro avg 0.50 0.50 0.50 2\n","weighted avg 0.50 0.50 0.50 2\n","\n"]}]},{"cell_type":"markdown","source":["seqeval은 리스트의 리스트로 구성된 예측과 레이블을 입력받음. 이런 지표를 훈련 과정에 통합하려면 모델 출력을 seqeval가 기대하는 리스트로 변환하는 함수가 필요. 이를 위해 연속된 부분단어의 레이블 ID는 무시하는 작업이 필요"],"metadata":{"id":"i0OmBlByZSDh"}},{"cell_type":"code","source":["import numpy as np\n","\n","def align_predictions(predictions, label_ids):\n"," preds = np.argmax(predictions, axis=2)\n"," batch_size, seq_len = preds.shape\n"," labels_list, preds_list = [], []\n","\n"," for batch_idx in range(batch_size):\n"," example_labels, example_preds = [], []\n"," for seq_idx in range(seq_len):\n"," # 레이블 IDs = -100 무시\n"," if label_ids[batch_idx, seq_idx] != -100:\n"," example_labels.append(index2tag[label_ids[batch_idx][seq_idx]])\n"," example_preds.append(index2tag[preds[batch_idx][seq_idx]])\n","\n"," labels_list.append(example_labels)\n"," preds_list.append(example_preds)\n","\n"," return preds_list, labels_list"],"metadata":{"id":"4jybU_ppZhRQ","executionInfo":{"status":"ok","timestamp":1675821120368,"user_tz":-540,"elapsed":3,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":34,"outputs":[]},{"cell_type":"markdown","source":["# XLM-RoBERTa 미세튜닝\n","PAN-X의 독일어 서브셋에 베이스 모델을 미세튜닝하고 프랑스어, 이태리어, 영어에서 제로샷 교차 언어 성능을 평가. 트랜스포머스 Trainer를 사용해 훈련 루프 처리"],"metadata":{"id":"U6UChFKpa9iq"}},{"cell_type":"code","source":["from transformers import TrainingArguments\n","\n","num_epochs = 3\n","batch_size = 24\n","logging_steps = len(panx_de_encoded[\"train\"])\n","model_name = f\"{xlmr_model_name}-finetuned-panx-de\"\n","training_args = TrainingArguments(output_dir=model_name, \n"," log_level=\"error\",\n"," num_train_epochs=num_epochs,\n"," per_device_train_batch_size=batch_size,\n"," per_device_eval_batch_size=batch_size,\n"," evaluation_strategy=\"epoch\",\n"," save_steps=1e6,\n"," weight_decay=0.01,\n"," disable_tqdm=False,\n"," logging_steps=logging_steps,\n"," push_to_hub=False)"],"metadata":{"id":"XdWVVYu5bJ9a","executionInfo":{"status":"ok","timestamp":1675821120368,"user_tz":-540,"elapsed":3,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":35,"outputs":[]},{"cell_type":"markdown","source":["검증 세트에서 평가 지표를 어떻게 계산해야 할지 Trainer로 전달해야 함 "],"metadata":{"id":"gg6J8oMzcT5y"}},{"cell_type":"code","source":["from seqeval.metrics import f1_score\n","\n","def compute_metrics(eval_pred):\n"," y_pred, y_true = align_predictions(eval_pred.predictions, eval_pred.label_ids)\n"," return {\"f1\": f1_score(y_true, y_pred)}"],"metadata":{"id":"i39ycBuhcC9e","executionInfo":{"status":"ok","timestamp":1675821120368,"user_tz":-540,"elapsed":3,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":36,"outputs":[]},{"cell_type":"markdown","source":["배치에서 가장 큰 시퀀스 길이로 입력 시퀀스를 패딩하도록 데이터 콜레이터(data collator)를 정의. 트랜스포머스는 토큰 분류를 위해 입력과 레이블을 패딩하는 전용 데이터 콜레이터를 제공"],"metadata":{"id":"Lhwpoa-Pcd1s"}},{"cell_type":"code","source":["from transformers import DataCollatorForTokenClassification\n","\n","data_collator = DataCollatorForTokenClassification(xlmr_tokenizer)"],"metadata":{"id":"lLfASUe6cmtH","executionInfo":{"status":"ok","timestamp":1675821120368,"user_tz":-540,"elapsed":2,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":37,"outputs":[]},{"cell_type":"markdown","source":["레이블도 시퀀스이기에 레이블 패딩도 필수"],"metadata":{"id":"F5mwIWiPcz8e"}},{"cell_type":"markdown","source":["Trainer를 위해 매번 새로운 모델을 만들지 않도록 model_init() 함수 제작. train() 메서드를 호출시 이 함수가 호출되어 훈련되지 않은 모델을 로드"],"metadata":{"id":"OiTxzDyAc4NX"}},{"cell_type":"code","source":["def model_init():\n"," return (XLMRobertaForTokenClassification.from_pretrained(xlmr_model_name, config=xlmr_config).to(device))"],"metadata":{"id":"fHxrwxLYdChA","executionInfo":{"status":"ok","timestamp":1675821120368,"user_tz":-540,"elapsed":2,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":38,"outputs":[]},{"cell_type":"markdown","source":["Encoding된 데이터셋과 함께 모든 정보를 Trainer에 전달"],"metadata":{"id":"MPGRlJimdoLF"}},{"cell_type":"code","source":["from transformers import Trainer\n","\n","trainer = Trainer(model_init=model_init,\n"," args=training_args,\n"," data_collator=data_collator,\n"," compute_metrics=compute_metrics,\n"," train_dataset=panx_de_encoded[\"train\"],\n"," eval_dataset=panx_de_encoded[\"validation\"],\n"," tokenizer=xlmr_tokenizer)"],"metadata":{"id":"FkzonrDedOZY","executionInfo":{"status":"ok","timestamp":1675821123421,"user_tz":-540,"elapsed":3055,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":39,"outputs":[]},{"cell_type":"code","source":["trainer.train()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":259},"id":"dp6ZZb8RdswR","executionInfo":{"status":"ok","timestamp":1675821640752,"user_tz":-540,"elapsed":517333,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"ffbdff70-13ec-4e23-afff-4cbbcfcb0248"},"execution_count":40,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.8/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n"," warnings.warn(\n"]},{"output_type":"display_data","data":{"text/plain":["\n"," \n","
\n"," \n"," \n"," \n"," Epoch \n"," Training Loss \n"," Validation Loss \n"," F1 \n"," \n"," \n"," 1 \n"," No log \n"," 0.162139 \n"," 0.820606 \n"," \n"," \n"," 2 \n"," No log \n"," 0.137930 \n"," 0.848582 \n"," \n"," \n"," \n","3 \n"," No log \n"," 0.135794 \n"," 0.863830 \n"," ▁Jeff ▁De an ▁ist ▁ein ▁Informati ker ▁bei ▁Google \n","Tags O B-PER I-PER I-PER O O O O O B-ORG \n","\n"," 10 11 12 13 \n","Tokens ▁in ▁Kaliforni en \n","Tags O B-LOC I-LOC I-PER "],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," 10 \n"," 11 \n"," 12 \n"," 13 \n"," \n"," \n"," Tokens \n"," <s> \n"," ▁Jeff \n"," ▁De \n"," an \n"," ▁ist \n"," ▁ein \n"," ▁Informati \n"," ker \n"," ▁bei \n"," ▁Google \n"," ▁in \n"," ▁Kaliforni \n"," en \n"," </s> \n"," \n"," \n"," \n","Tags \n"," O \n"," B-PER \n"," I-PER \n"," I-PER \n"," O \n"," O \n"," O \n"," O \n"," O \n"," B-ORG \n"," O \n"," B-LOC \n"," I-LOC \n"," I-PER \n","
\n","\n","모델의 성능이 기대에 못 미칠 때 오류를 살펴보면, 유용한 통찰을 얻고 코드만 봐서는 찾기 힘든 버그를 발견하게 됨.
\n","오류 분석을 위한 방법으로 **손실이 가장 큰 검증 샘플을 살펴보는 것**. 샘플 시퀀스의 토큰마다 손실을 계산"],"metadata":{"id":"kNA8C7eHhPfM"}},{"cell_type":"code","source":["from torch.nn.functional import cross_entropy\n","\n","def forward_pass_with_label(batch):\n"," # 리스트의 딕셔너리를 데이터 콜레이터에 적합한 딕셔너리의 리스트로 변환\n"," features = [dict(zip(batch, t)) for t in zip(*batch.values())]\n"," # 입력과 레이블을 패딩하고 모든 텐서를 장치에 배치\n"," batch = data_collator(features)\n"," input_ids = batch[\"input_ids\"].to(device)\n"," attention_mask = batch[\"attention_mask\"].to(device)\n"," labels = batch[\"labels\"].to(device)\n"," with torch.no_grad():\n"," # 데이터를 모델에 전달\n"," output = trainer.model(input_ids, attention_mask)\n"," # logit.size: [batch_size, sequence_length, classes]\n"," # 마지막 축을 따라 가장 큰 로짓 값을 가진 클래스를 선택\n"," predicted_label = torch.argmax(output.logits, axis=-1).cpu().numpy()\n"," # 배치 차원을 펼친 다음 토큰마다 손실을 계산\n"," loss = cross_entropy(output.logits.view(-1, 7), labels.view(-1), reduction=\"none\")\n"," # 배치 차원을 다시 만들고 넘파이 배열로 변환\n"," loss = loss.view(len(input_ids), -1).cpu().numpy()\n","\n"," return {\"loss\":loss, \"predicted_label\": predicted_label}"],"metadata":{"id":"6vAZdu1Nh1GH","executionInfo":{"status":"ok","timestamp":1675821640752,"user_tz":-540,"elapsed":7,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":42,"outputs":[]},{"cell_type":"code","source":["valid_set = panx_de_encoded[\"validation\"]\n","valid_set = valid_set.map(forward_pass_with_label, batched=True, batch_size=32)\n","df = valid_set.to_pandas()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":86,"referenced_widgets":["4cc6e9d90ff749db80a80877c4ceb88b","898cd3c6298d42dc9d0bd51adcee9069","c3597b8ec6734c35b028daea5e4a3bbb","31d42d93cd534bb09fc0d962b63ae09e","84ef7a7c8fc342d48da529843f65f2fa","e1fadba579f04906bcf21ff823e366f9","b9c6e83d7b8f4ec48de17c75776fbf59","17d21e082a4e460c956158a271ad48ee","42c79b0e7b364da98319a4cd8522eb13","fbe1408c5e624cc9becac343bcbdfff0","0a17783cd6264c85b3d75c6cbfc189b3"]},"id":"7HHXhKiNq0h5","executionInfo":{"status":"ok","timestamp":1675821669245,"user_tz":-540,"elapsed":28500,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"8a803f54-b85a-4955-8245-fb8242ef7fb7"},"execution_count":43,"outputs":[{"output_type":"stream","name":"stderr","text":["WARNING:datasets.fingerprint:Parameter 'function'=\n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," input_ids \n"," attention_mask \n"," labels \n"," loss \n"," predicted_label \n"," input_tokens \n"," \n"," \n"," \n","0 \n"," [0, 10699, 11, 15, 16104, 1388, 2] \n"," [1, 1, 1, 1, 1, 1, 1] \n"," [IGN, B-ORG, IGN, I-ORG, I-ORG, I-ORG, IGN] \n"," [0.0, 0.01121267, 0.0, 0.012447174, 0.01018930... \n"," [I-ORG, B-ORG, I-ORG, I-ORG, I-ORG, I-ORG, I-ORG] \n"," [<s>, ▁Ham, a, ▁(, ▁Unternehmen, ▁), </s>] \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," input_ids \n"," attention_mask \n"," labels \n"," loss \n"," predicted_label \n"," input_tokens \n"," \n"," \n"," 0 \n"," 10699 \n"," 1 \n"," B-ORG \n"," 0.01 \n"," B-ORG \n"," ▁Ham \n"," \n"," \n"," 0 \n"," 15 \n"," 1 \n"," I-ORG \n"," 0.01 \n"," I-ORG \n"," ▁( \n"," \n"," \n"," 0 \n"," 16104 \n"," 1 \n"," I-ORG \n"," 0.01 \n"," I-ORG \n"," ▁Unternehmen \n"," \n"," \n"," 0 \n"," 1388 \n"," 1 \n"," I-ORG \n"," 0.01 \n"," I-ORG \n"," ▁) \n"," \n"," \n"," 1 \n"," 56530 \n"," 1 \n"," O \n"," 0.00 \n"," O \n"," ▁WE \n"," \n"," \n"," 1 \n"," 83982 \n"," 1 \n"," B-ORG \n"," 2.90 \n"," B-PER \n"," ▁Luz \n"," \n"," \n"," \n","1 \n"," 10 \n"," 1 \n"," I-ORG \n"," 2.83 \n"," I-PER \n"," ▁a \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," \n"," \n"," input_tokens \n"," ▁ \n"," ▁in \n"," ▁der \n"," ▁von \n"," ▁und \n"," ▁/ \n"," ▁'' \n"," ▁( \n"," ▁) \n"," ▁A \n"," \n"," \n"," count \n"," 6066 \n"," 989 \n"," 1388 \n"," 808 \n"," 1171 \n"," 163 \n"," 2898 \n"," 246 \n"," 246 \n"," 125 \n"," \n"," \n"," mean \n"," 0.03 \n"," 0.14 \n"," 0.09 \n"," 0.13 \n"," 0.07 \n"," 0.46 \n"," 0.02 \n"," 0.28 \n"," 0.26 \n"," 0.46 \n"," \n"," \n"," \n","sum \n"," 203.8 \n"," 143.31 \n"," 129.16 \n"," 105.29 \n"," 81.04 \n"," 75.43 \n"," 70.93 \n"," 69.69 \n"," 63.74 \n"," 57.43 \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," \n"," \n"," labels \n"," B-ORG \n"," I-LOC \n"," I-ORG \n"," B-LOC \n"," B-PER \n"," I-PER \n"," O \n"," \n"," \n"," count \n"," 2683 \n"," 1462 \n"," 3820 \n"," 3172 \n"," 2893 \n"," 4139 \n"," 43648 \n"," \n"," \n"," mean \n"," 0.62 \n"," 0.61 \n"," 0.48 \n"," 0.33 \n"," 0.27 \n"," 0.19 \n"," 0.03 \n"," \n"," \n"," \n","sum \n"," 1675.56 \n"," 897.82 \n"," 1826.57 \n"," 1035.34 \n"," 792.12 \n"," 786.43 \n"," 1337.71 \n","
\n","explode() 함수를 적용하기 전의 원래 DataFrame을 사용하여 토큰당 손실을 합산해 총 손실을 계산"],"metadata":{"id":"jIxU_vpQy7nn"}},{"cell_type":"code","source":["def get_samples(df):\n"," for _, row in df.iterrows():\n"," labels, preds, tokens, losses = [], [], [], []\n"," for i, mask in enumerate(row[\"attention_mask\"]):\n"," if i not in {0, len(row[\"attention_mask\"])}:\n"," labels.append(row[\"labels\"][i])\n"," preds.append(row[\"predicted_label\"][i])\n"," tokens.append(row[\"input_tokens\"][i])\n"," losses.append(f\"{row['loss'][i]:.2f}\")\n"," df_tmp = pd.DataFrame({\"tokens\": tokens, \"labels\": labels, \"preds\": preds, \"losses\": losses}).T\n"," yield df_tmp\n","\n","df[\"total_loss\"] = df[\"loss\"].apply(sum)\n","df_tmp = df.sort_values(by=\"total_loss\", ascending=False).head(3)\n","\n","for sample in get_samples(df_tmp):\n"," display(sample)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":490},"id":"gQ8-n0qQzErD","executionInfo":{"status":"ok","timestamp":1675822882048,"user_tz":-540,"elapsed":789,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"14c60418-8e39-4c8b-83de-e1190849dea6"},"execution_count":54,"outputs":[{"output_type":"display_data","data":{"text/plain":[" 0 1 2 3 4 5 6 7 8 9 \\\n","tokens ▁' ▁'' ▁Τ Κ ▁'' ▁' ▁' ▁'' ▁T ▁'' \n","labels O O O IGN O O B-LOC I-LOC I-LOC I-LOC \n","preds O O B-ORG B-ORG O O O O B-ORG O \n","losses 0.00 0.00 3.44 0.00 0.00 0.00 10.46 10.07 8.27 9.45 \n","\n"," 10 11 12 13 14 15 16 17 18 \n","tokens ▁' ri ▁'' ▁' k ▁'' ▁' ala \n","labels I-LOC IGN I-LOC I-LOC IGN I-LOC I-LOC IGN IGN \n","preds O O O O O O O O O \n","losses 10.15 0.00 9.87 10.14 0.00 10.09 10.17 0.00 0.00 "],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," 10 \n"," 11 \n"," 12 \n"," 13 \n"," 14 \n"," 15 \n"," 16 \n"," 17 \n"," 18 \n"," \n"," \n"," tokens \n"," ▁' \n"," ▁'' \n"," ▁Τ \n"," Κ \n"," ▁'' \n"," ▁' \n"," ▁' \n"," ▁'' \n"," ▁T \n"," ▁'' \n"," ▁' \n"," ri \n"," ▁'' \n"," ▁' \n"," k \n"," ▁'' \n"," ▁' \n"," ala \n"," </s> \n"," \n"," \n"," labels \n"," O \n"," O \n"," O \n"," IGN \n"," O \n"," O \n"," B-LOC \n"," I-LOC \n"," I-LOC \n"," I-LOC \n"," I-LOC \n"," IGN \n"," I-LOC \n"," I-LOC \n"," IGN \n"," I-LOC \n"," I-LOC \n"," IGN \n"," IGN \n"," \n"," \n"," preds \n"," O \n"," O \n"," B-ORG \n"," B-ORG \n"," O \n"," O \n"," O \n"," O \n"," B-ORG \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," \n"," \n"," \n","losses \n"," 0.00 \n"," 0.00 \n"," 3.44 \n"," 0.00 \n"," 0.00 \n"," 0.00 \n"," 10.46 \n"," 10.07 \n"," 8.27 \n"," 9.45 \n"," 10.15 \n"," 0.00 \n"," 9.87 \n"," 10.14 \n"," 0.00 \n"," 10.09 \n"," 10.17 \n"," 0.00 \n"," 0.00 \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," 10 \n"," 11 \n"," 12 \n"," 13 \n"," 14 \n"," 15 \n"," 16 \n"," 17 \n"," \n"," \n"," tokens \n"," ▁'' \n"," 8 \n"," . \n"," ▁Juli \n"," ▁'' \n"," ▁: \n"," ▁Protest \n"," camp \n"," ▁auf \n"," ▁dem \n"," ▁Gelände \n"," ▁der \n"," ▁Republika \n"," n \n"," ischen \n"," ▁Gar \n"," de \n"," </s> \n"," \n"," \n"," labels \n"," B-ORG \n"," IGN \n"," IGN \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," IGN \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," IGN \n"," IGN \n"," I-ORG \n"," IGN \n"," IGN \n"," \n"," \n"," preds \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," O \n"," B-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," O \n"," \n"," \n"," \n","losses \n"," 8.55 \n"," 0.00 \n"," 0.00 \n"," 8.53 \n"," 7.80 \n"," 8.47 \n"," 7.80 \n"," 0.00 \n"," 9.04 \n"," 9.79 \n"," 8.70 \n"," 7.25 \n"," 5.87 \n"," 0.00 \n"," 0.00 \n"," 0.01 \n"," 0.00 \n"," 0.00 \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," 10 \n"," 11 \n"," 12 \n"," 13 \n"," 14 \n"," \n"," \n"," tokens \n"," ▁United \n"," ▁Nations \n"," ▁Multi \n"," dimensional \n"," ▁Integra \n"," ted \n"," ▁Stabil \n"," ization \n"," ▁Mission \n"," ▁in \n"," ▁the \n"," ▁Central \n"," ▁African \n"," ▁Republic \n"," </s> \n"," \n"," \n"," labels \n"," B-PER \n"," I-PER \n"," I-PER \n"," IGN \n"," I-PER \n"," IGN \n"," I-PER \n"," IGN \n"," I-PER \n"," I-PER \n"," I-PER \n"," I-PER \n"," I-PER \n"," I-PER \n"," IGN \n"," \n"," \n"," preds \n"," B-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," I-ORG \n"," \n"," \n"," \n","losses \n"," 6.26 \n"," 5.70 \n"," 6.25 \n"," 0.00 \n"," 6.13 \n"," 0.00 \n"," 5.89 \n"," 0.00 \n"," 5.74 \n"," 5.50 \n"," 5.81 \n"," 5.95 \n"," 5.88 \n"," 5.63 \n"," 0.00 \n","
\n","PAN-X 데이터셋의 레이블이 자동으로 생성된것으로 보임. 이런 레이블을 '실버 스탠다드(silver standard)'라 함(사람이 생성한 레이블은 골드 스탠다드(gold standard)라 함)\n","
\n","괄호와 슬래시의 손실이 비교적 높음"],"metadata":{"id":"pO2taj-T1C9r"}},{"cell_type":"markdown","source":["# 교차 언어 전이\n","독일어에서 XLM-R을 미세 튜닝 했으므로 Trainer 클래스의 predict() 메서드를 사용해 다른 언어로 전이되는 능력을 평가"],"metadata":{"id":"J2fULCVG11Ff"}},{"cell_type":"code","source":["def get_f1_score(trainer, dataset):\n"," return trainer.predict(dataset).metrics[\"test_f1\"]"],"metadata":{"id":"HilLTf3k19gR","executionInfo":{"status":"ok","timestamp":1675823275711,"user_tz":-540,"elapsed":514,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":55,"outputs":[]},{"cell_type":"markdown","source":["테스트 세트의 성능을 평가"],"metadata":{"id":"rAGw8lNS2gwz"}},{"cell_type":"code","source":["f1_scores = defaultdict(dict)\n","f1_scores[\"de\"][\"de\"] = get_f1_score(trainer, panx_de_encoded[\"test\"])\n","print(f\"[de] 데이터셋에서 [de] 모델의 F1 점수: {f1_scores['de']['de']:.3f}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"id":"3Xxms1Eq2i85","executionInfo":{"status":"ok","timestamp":1675823425440,"user_tz":-540,"elapsed":18895,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"9b8c1ea4-a2a2-4d2d-f01e-af6e9398b8a6"},"execution_count":57,"outputs":[{"output_type":"display_data","data":{"text/plain":[" ▁Jeff ▁De an ▁est ▁informatic ien ▁chez ▁Google \n","Tags I-PER B-PER I-PER I-PER O O O O B-ORG \n","\n"," 9 10 11 12 13 \n","Tokens ▁en ▁cali for nie \n","Tags O B-LOC I-LOC I-LOC I-PER "],"text/html":["\n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," 0 \n"," 1 \n"," 2 \n"," 3 \n"," 4 \n"," 5 \n"," 6 \n"," 7 \n"," 8 \n"," 9 \n"," 10 \n"," 11 \n"," 12 \n"," 13 \n"," \n"," \n"," Tokens \n"," <s> \n"," ▁Jeff \n"," ▁De \n"," an \n"," ▁est \n"," ▁informatic \n"," ien \n"," ▁chez \n"," ▁Google \n"," ▁en \n"," ▁cali \n"," for \n"," nie \n"," </s> \n"," \n"," \n"," \n","Tags \n"," I-PER \n"," B-PER \n"," I-PER \n"," I-PER \n"," O \n"," O \n"," O \n"," O \n"," B-ORG \n"," O \n"," B-LOC \n"," I-LOC \n"," I-LOC \n"," I-PER \n"," \n"," \n","
\n"," \n"," \n"," \n"," Epoch \n"," Training Loss \n"," Validation Loss \n"," F1 \n"," \n"," \n"," 1 \n"," 1.797900 \n"," 1.382164 \n"," 0.110526 \n"," \n"," \n"," 2 \n"," 1.306500 \n"," 1.131393 \n"," 0.170438 \n"," \n"," \n"," \n","3 \n"," 1.084300 \n"," 1.035612 \n"," 0.185041 \n"," \n"," \n","
\n","\n"," \n"," \n"," \n"," \n"," num_samples \n"," f1_score \n"," \n"," \n"," \n","0 \n"," 250 \n"," 0.167937 \n","
\n","\n","훈련 세트 크기를 늘릴 경우 확인"],"metadata":{"id":"qgCQcAHU6gzI"}},{"cell_type":"code","source":["for num_samples in [500, 1000, 2000, 4000]:\n"," metrics_df = metrics_df.append(train_on_subset(panx_fr_encoded, num_samples), ignore_index=True)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":751},"id":"PaHHmz6Z6on2","executionInfo":{"status":"ok","timestamp":1675824780020,"user_tz":-540,"elapsed":349629,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"37a1da11-39c5-4708-d265-60192b1b6daf"},"execution_count":66,"outputs":[{"output_type":"stream","name":"stderr","text":["WARNING:datasets.arrow_dataset:Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/29f5d57a48779f37ccb75cb8708d1095448aad0713b425bdc1ff9a4a128a56e4/cache-6b17d58cd0d6e4ba.arrow\n","/usr/local/lib/python3.8/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n"," warnings.warn(\n"]},{"output_type":"display_data","data":{"text/plain":["\n"," \n","
\n"," \n"," \n"," \n"," Epoch \n"," Training Loss \n"," Validation Loss \n"," F1 \n"," \n"," \n"," 1 \n"," 1.492500 \n"," 1.051098 \n"," 0.185071 \n"," \n"," \n"," 2 \n"," 0.899900 \n"," 0.705176 \n"," 0.569553 \n"," \n"," \n"," \n","3 \n"," 0.587300 \n"," 0.584122 \n"," 0.618641 \n"," \n"," \n","
\n"," \n"," \n"," \n"," Epoch \n"," Training Loss \n"," Validation Loss \n"," F1 \n"," \n"," \n"," 1 \n"," 1.205300 \n"," 0.606873 \n"," 0.550045 \n"," \n"," \n"," 2 \n"," 0.481400 \n"," 0.425944 \n"," 0.702297 \n"," \n"," \n"," \n","3 \n"," 0.330100 \n"," 0.390302 \n"," 0.704059 \n"," \n"," \n","
\n"," \n"," \n"," \n"," Epoch \n"," Training Loss \n"," Validation Loss \n"," F1 \n"," \n"," \n"," 1 \n"," 0.805700 \n"," 0.408441 \n"," 0.737765 \n"," \n"," \n"," 2 \n"," 0.331000 \n"," 0.350425 \n"," 0.780576 \n"," \n"," \n"," \n","3 \n"," 0.217100 \n"," 0.333629 \n"," 0.810279 \n"," \n"," \n","
\n"," \n"," \n"," \n"," Epoch \n"," Training Loss \n"," Validation Loss \n"," F1 \n"," \n"," \n"," 1 \n"," 0.605400 \n"," 0.337464 \n"," 0.781681 \n"," \n"," \n"," 2 \n"," 0.271200 \n"," 0.285458 \n"," 0.813073 \n"," \n"," \n"," \n","3 \n"," 0.182100 \n"," 0.287294 \n"," 0.833361 \n","
\n","문서를 일일이 레이블링 하는 비용을 생각했을때 제로샷이 경제적으로 더 우위를 점하고 괜찮음"],"metadata":{"id":"JIalkzgV7gFI"}},{"cell_type":"markdown","source":["# 다국어에서 동시에 미세튜닝하기\n","성능 감소 폭을 줄이는 방법 하나는 다국어에서 동시에 미세튜닝을 하는것"],"metadata":{"id":"sOkBGZr-7_Pv"}},{"cell_type":"code","source":["from datasets import concatenate_datasets\n","\n","def concatenate_splits(corpora):\n"," multi_corpus = DatasetDict()\n"," for split in corpora[0].keys():\n"," multi_corpus[split] = concatenate_datasets([corpus[split] for corpus in corpora]).shuffle(seed=42)\n"," return multi_corpus\n","\n","panx_de_fr_encoded = concatenate_splits([panx_de_encoded, panx_fr_encoded])"],"metadata":{"id":"Vvfd7NRp8ElJ","executionInfo":{"status":"ok","timestamp":1675824873243,"user_tz":-540,"elapsed":2,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}}},"execution_count":68,"outputs":[]},{"cell_type":"code","source":["training_args.logging_steps = len(panx_de_fr_encoded[\"train\"]) // batch_size\n","training_args.push_to_hub = False\n","training_args.output_dir = \"xlm-roberta-base-finetuned-panx-de-fr\"\n","\n","trainer = Trainer(model_init=model_init,\n"," args=training_args,\n"," data_collator=data_collator,\n"," compute_metrics=compute_metrics,\n"," tokenizer=xlmr_tokenizer,\n"," train_dataset=panx_de_fr_encoded[\"train\"],\n"," eval_dataset=panx_de_fr_encoded[\"validation\"])\n","\n","trainer.train()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":259},"id":"prfwfMh68nhG","executionInfo":{"status":"ok","timestamp":1675825757936,"user_tz":-540,"elapsed":700206,"user":{"displayName":"HanGyo Jung","userId":"11224950994115057744"}},"outputId":"db544003-a225-4230-81f4-0f0c4c931d26"},"execution_count":69,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.8/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n"," warnings.warn(\n"]},{"output_type":"display_data","data":{"text/plain":["\n"," \n","
\n"," \n"," \n"," \n"," Epoch \n"," Training Loss \n"," Validation Loss \n"," F1 \n"," \n"," \n"," 1 \n"," 0.290500 \n"," 0.178259 \n"," 0.830973 \n"," \n"," \n"," 2 \n"," 0.146100 \n"," 0.159958 \n"," 0.845506 \n"," \n"," \n"," \n","3 \n"," 0.094800 \n"," 0.165646 \n"," 0.858905 \n"," \n",
+ " \n",
+ "
\n",
+ ""
+ ],
+ "text/plain": [
+ " text label\n",
+ "0 i didnt feel humiliated 0\n",
+ "1 i can go from feeling so hopeless to so damned... 0\n",
+ "2 im grabbing a minute to post i feel greedy wrong 3\n",
+ "3 i am ever feeling nostalgic about the fireplac... 2\n",
+ "4 i am feeling grouchy 3"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "emotions.set_format(type=\"pandas\")\n",
+ "df = emotions[\"train\"][:]\n",
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "44b1fd35",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " text \n",
+ " label \n",
+ " \n",
+ " \n",
+ " 0 \n",
+ " i didnt feel humiliated \n",
+ " 0 \n",
+ " \n",
+ " \n",
+ " 1 \n",
+ " i can go from feeling so hopeless to so damned... \n",
+ " 0 \n",
+ " \n",
+ " \n",
+ " 2 \n",
+ " im grabbing a minute to post i feel greedy wrong \n",
+ " 3 \n",
+ " \n",
+ " \n",
+ " 3 \n",
+ " i am ever feeling nostalgic about the fireplac... \n",
+ " 2 \n",
+ " \n",
+ " \n",
+ " \n",
+ "4 \n",
+ " i am feeling grouchy \n",
+ " 3 \n",
+ " \n",
+ " \n",
+ "
\n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " text \n",
+ " label \n",
+ " label_name \n",
+ " \n",
+ " \n",
+ " 0 \n",
+ " i didnt feel humiliated \n",
+ " 0 \n",
+ " sadness \n",
+ " \n",
+ " \n",
+ " 1 \n",
+ " i can go from feeling so hopeless to so damned... \n",
+ " 0 \n",
+ " sadness \n",
+ " \n",
+ " \n",
+ " 2 \n",
+ " im grabbing a minute to post i feel greedy wrong \n",
+ " 3 \n",
+ " anger \n",
+ " \n",
+ " \n",
+ " 3 \n",
+ " i am ever feeling nostalgic about the fireplac... \n",
+ " 2 \n",
+ " love \n",
+ " \n",
+ " \n",
+ " \n",
+ "4 \n",
+ " i am feeling grouchy \n",
+ " 3 \n",
+ " anger \n",
+ "
\n",
+ "파이썬의 str 객체 내부는 사실 배열이므로 문자 수준의 토큰화가 손쉽게 가능"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "9b5c18a4",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['T', 'o', 'k', 'e', 'n', 'i', 'z', 'i', 'n', 'g', ' ', 't', 'e', 'x', 't', ' ', 'i', 's', ' ', 'a', ' ', 'c', 'o', 'r', 'e', ' ', 't', 'a', 's', 'k', ' ', 'o', 'f', ' ', 'N', 'L', 'P', '.']\n"
+ ]
+ }
+ ],
+ "source": [
+ "text = \"Tokenizing text is a core task of NLP.\"\n",
+ "tokenized_text = list(text)\n",
+ "print(tokenized_text)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "acf0acdd",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{' ': 0, '.': 1, 'L': 2, 'N': 3, 'P': 4, 'T': 5, 'a': 6, 'c': 7, 'e': 8, 'f': 9, 'g': 10, 'i': 11, 'k': 12, 'n': 13, 'o': 14, 'r': 15, 's': 16, 't': 17, 'x': 18, 'z': 19}\n"
+ ]
+ }
+ ],
+ "source": [
+ "token2idx = {ch: idx for idx, ch in enumerate(sorted(set(tokenized_text)))}\n",
+ "print(token2idx)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "id": "ff118c22",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[5, 14, 12, 8, 13, 11, 19, 11, 13, 10, 0, 17, 8, 18, 17, 0, 11, 16, 0, 6, 0, 7, 14, 15, 8, 0, 17, 6, 16, 12, 0, 14, 9, 0, 3, 2, 4, 1]\n"
+ ]
+ }
+ ],
+ "source": [
+ "input_ids = [token2idx[token] for token in tokenized_text]\n",
+ "print(input_ids)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "id": "8508c09d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch\n",
+ "import torch.nn.functional as F"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "id": "4ce2e4e1",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/var/folders/rz/96n_vjq17kg53f0rgc21grc40000gn/T/ipykernel_1858/3674191911.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+ " input_ids = torch.tensor(input_ids)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([38, 20])"
+ ]
+ },
+ "execution_count": 54,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "input_ids = torch.tensor(input_ids)\n",
+ "one_hot_encodings = F.one_hot(input_ids, num_classes=len(token2idx))\n",
+ "one_hot_encodings.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8b8449c2",
+ "metadata": {},
+ "source": [
+ "38개의 입력 토큰 각각에 20차원의 원-핫 벡터가 만들어짐\n",
+ "\n",
+ "5 .....len(20)
\n",
+ "14.....len(20)
\n",
+ "12.....len(20)
\n",
+ ".\n",
+ ".\n",
+ ". .....len(20)
\n",
+ "2 .....len(20)
\n",
+ "4 .....len(20)
\n",
+ "1 .....len(20)
\n",
+ "\n",
+ "각 토큰별 20개의 원소를 가진 벡터가 만들어짐"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "id": "58874e4b",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "토큰: Tokenizing\n",
+ "텐서 인덱스: 5\n",
+ "원-핫 인코딩: tensor([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(f\"토큰: {tokenized_text[0]}\")\n",
+ "print(f\"텐서 인덱스: {input_ids[0]}\")\n",
+ "print(f\"원-핫 인코딩: {one_hot_encodings[0]}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5729671a",
+ "metadata": {},
+ "source": [
+ "## 단어 토큰화\n",
+ "텍스트를 문자가 아닌 단어로 분할하고 각 단어를 정수로 매핑"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "id": "6fbe209f",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['this', 'is', 'a', 'test']\n"
+ ]
+ }
+ ],
+ "source": [
+ "tokenized_text = text.split()\n",
+ "print(tokenized_text)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f8988f71",
+ "metadata": {},
+ "source": [
+ "단어 토큰화는 단어에 곡용, 활용형, 철자 오류가 포함되어 어휘 사전이 금세 수백만 개까지 늘어나게 됨"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bfb042f0",
+ "metadata": {},
+ "source": [
+ "단어를 토큰화 하게 되면 어휘사전이 커지게 되고 이러면 신경망의 파라미터 역시 많이 필요하게 됨\n",
+ "
\n",
+ "- 어휘 사전의 크기를 제한하는 일반적인 방법은 드물게 등장하는 단어는 무시하는 것. 이러한 단어는 'UNK' 토큰으로 매핑\n",
+ "- 하지만 이렇게 되면 토큰화 과정에서 중요 정보 일부를 잃게 됨\n",
+ "- 모든 입력 정보와 일부 입력 구조를 유지하는 문자 토큰화와 단어 토큰화를 절충하는 방법으로 **부분단어 토큰화(subword tokenization)** 라는 방법"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "95a96577",
+ "metadata": {},
+ "source": [
+ "## 부분단어 토큰화\n",
+ "- 부분단어 토큰화는 문자 단어 토큰화 + 단어 토큰화 의 장점을 결합\n",
+ "- 드물게 등장하는 단어를 더 작은 단위로 나누면 모델이 복잡한 단어나 철자 오류를 처리하기 용이\n",
+ "- 다른 방법으론 입력 길이를 적절한 크기로 유지하기 위해 자주 등장하는 단어를 고유한 항목으로 유지\n",
+ "
\n",
+ "- NLP 분야에서 널리 사용되는 부분단어 토큰화 중 먼저 BERT와 DistilBERT의 토크나이저 **WordPiece**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "id": "2379be10",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from transformers import AutoTokenizer\n",
+ "\n",
+ "model_ckpt = \"distilbert-base-uncased\"\n",
+ "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "id": "6b1e2b90",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "this is a test\n",
+ "{'input_ids': [101, 2023, 2003, 1037, 3231, 102], 'attention_mask': [1, 1, 1, 1, 1, 1]}\n"
+ ]
+ }
+ ],
+ "source": [
+ "encoded_text = tokenizer(text)\n",
+ "print(text)\n",
+ "print(encoded_text)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 59,
+ "id": "0b85a1b6",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['[CLS]', 'this', 'is', 'a', 'test', '[SEP]']\n"
+ ]
+ }
+ ],
+ "source": [
+ "tokens = tokenizer.convert_ids_to_tokens(encoded_text.input_ids)\n",
+ "print(tokens)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "94b6f7c5",
+ "metadata": {},
+ "source": [
+ "1. 특수 토큰 [CLS], [SEP]가 시퀀스 처음과 끝에 추가\n",
+ "2. 토큰이 모두 소문자로 변환\n",
+ "3. 'tokenizing'과 'NLP'가 각각 2개의 토큰으로 나뉘어짐. 자주 등장하는 단어가 아니기 때문 ##izing, ##p에 있는 #은 공백이 아님을 의미\n",
+ "\n",
+ "이런 접두사가 붙은 토큰은 문자열로 다시 바꿀 때 앞의 토큰과 합침"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "id": "259920c8",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[CLS] this is a test [SEP]\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(tokenizer.convert_tokens_to_string(tokens))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "id": "46c19084",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "30522"
+ ]
+ },
+ "execution_count": 61,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# 어휘 사전 크기 확인\n",
+ "tokenizer.vocab_size"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 62,
+ "id": "15e57040",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "512"
+ ]
+ },
+ "execution_count": 62,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# 모델 최대 문맥 크기\n",
+ "tokenizer.model_max_length"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 63,
+ "id": "dbbba4bb",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['input_ids', 'attention_mask']"
+ ]
+ },
+ "execution_count": 63,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# 모델이 정방향 패스(forward pass)\n",
+ "tokenizer.model_input_names"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f8fa19c9",
+ "metadata": {},
+ "source": [
+ "# 전체 데이터셋 토큰화\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
+ "id": "5e5a58d3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def tokenize(batch):\n",
+ " return tokenizer(batch[\"text\"], padding=True, truncation=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "db6da0cc",
+ "metadata": {},
+ "source": [
+ "Tokenizer를 샘플 배치에 적용\n",
+ "
\n",
+ "- padding=True: 배치에 있는 가장 긴 샘플 크기에 맞춰 샘플을 0으로 패딩\n",
+ "- truncation=True: 모델의 최대 문맥 크기에 맞춰 샘플을 잘라냄"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 65,
+ "id": "9658e034",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'input_ids': [[101, 1045, 2134, 2102, 2514, 26608, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1045, 2064, 2175, 2013, 3110, 2061, 20625, 2000, 2061, 9636, 17772, 2074, 2013, 2108, 2105, 2619, 2040, 14977, 1998, 2003, 8300, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(tokenize(emotions[\"train\"][:2]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 66,
+ "id": "03458041",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Loading cached processed dataset at /Users/hangyojeong/.cache/huggingface/datasets/emotion/split/1.0.0/cca5efe2dfeb58c1d098e0f9eeb200e9927d889b5a03c67097275dfb5fe463bd/cache-6316da5de5436960.arrow\n",
+ "Loading cached processed dataset at /Users/hangyojeong/.cache/huggingface/datasets/emotion/split/1.0.0/cca5efe2dfeb58c1d098e0f9eeb200e9927d889b5a03c67097275dfb5fe463bd/cache-2cb11f4abdb48d91.arrow\n",
+ "Loading cached processed dataset at /Users/hangyojeong/.cache/huggingface/datasets/emotion/split/1.0.0/cca5efe2dfeb58c1d098e0f9eeb200e9927d889b5a03c67097275dfb5fe463bd/cache-1983a40220248de7.arrow\n"
+ ]
+ }
+ ],
+ "source": [
+ "emotions_encoded = emotions.map(tokenize, batched=True, batch_size=None)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ceffe2fe",
+ "metadata": {},
+ "source": [
+ "map() 메서드는 말뭉치에 있는 모든 샘플에 개별적으로 작용
\n",
+ "\n",
+ "- batched=True: 트윗을 배치로 인코딩\n",
+ "- batch_size=None: 전체 데이터셋이 하나의 배치로 tokenize() 함수에 적용, 입력 텐서와 어텐션 마스크는 전역적으로 동일한 크기로 생성"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 67,
+ "id": "3de1ab5c",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['text', 'label', 'input_ids', 'attention_mask']\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(emotions_encoded[\"train\"].column_names)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c3d9224c",
+ "metadata": {},
+ "source": [
+ "# 텍스트 분류 모델 훈련\n",
+ "\n",
+ "### 특성 추출\n",
+ "사전 훈련된 모델을 수정하지 않고 은닉 상태를 특성(feature)으로 사용해 분류 모델을 훈련\n",
+ "\n",
+ "### 미세 튜닝\n",
+ "사전 훈련된 모델의 파라미터도 업데이트하기 위해 전체 모델을 엔드-투-엔드로 훈련"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2b923107",
+ "metadata": {},
+ "source": [
+ "## 특성 추출 방법\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c6560f7f",
+ "metadata": {},
+ "source": [
+ "사전 훈련된 모델 사용하기"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 68,
+ "id": "e6f48da5",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight']\n",
+ "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+ "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+ ]
+ }
+ ],
+ "source": [
+ "from transformers import AutoModel\n",
+ "\n",
+ "model_ckpt = \"distilbert-base-uncased\"\n",
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ "model = AutoModel.from_pretrained(model_ckpt).to(device)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 69,
+ "id": "5e3da298",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "입력 텐서 크기: torch.Size([1, 6])\n"
+ ]
+ }
+ ],
+ "source": [
+ "text = \"this is a test\"\n",
+ "inputs = tokenizer(text, return_tensors=\"pt\")\n",
+ "print(f\"입력 텐서 크기: {inputs['input_ids'].size()}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 70,
+ "id": "4f99ad6f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_items([('input_ids', tensor([[ 101, 2023, 2003, 1037, 3231, 102]])), ('attention_mask', tensor([[1, 1, 1, 1, 1, 1]]))])"
+ ]
+ },
+ "execution_count": 70,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "inputs.items()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 71,
+ "id": "a81ee80b",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "k: input_ids\n",
+ "v: tensor([[ 101, 2023, 2003, 1037, 3231, 102]])\n",
+ "k: attention_mask\n",
+ "v: tensor([[1, 1, 1, 1, 1, 1]])\n"
+ ]
+ }
+ ],
+ "source": [
+ "for k, v in inputs.items():\n",
+ " print(f\"k: {k}\")\n",
+ " print(f\"v: {v}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 72,
+ "id": "c3d5a3ca",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "BaseModelOutput(last_hidden_state=tensor([[[-0.1565, -0.1862, 0.0528, ..., -0.1188, 0.0662, 0.5470],\n",
+ " [-0.3575, -0.6484, -0.0618, ..., -0.3040, 0.3508, 0.5221],\n",
+ " [-0.2772, -0.4459, 0.1818, ..., -0.0948, -0.0076, 0.9958],\n",
+ " [-0.2841, -0.3917, 0.3753, ..., -0.2151, -0.1173, 1.0526],\n",
+ " [ 0.2661, -0.5094, -0.3180, ..., -0.4203, 0.0144, -0.2149],\n",
+ " [ 0.9441, 0.0112, -0.4714, ..., 0.1439, -0.7288, -0.1619]]]), hidden_states=None, attentions=None)\n"
+ ]
+ }
+ ],
+ "source": [
+ "inputs = {k:v.to(device) for k,v in inputs.items()}\n",
+ "with torch.no_grad():\n",
+ " outputs = model(**inputs)\n",
+ " \n",
+ "print(outputs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 73,
+ "id": "a7b6ba5d",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([1, 6, 768])"
+ ]
+ },
+ "execution_count": 73,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "outputs.last_hidden_state.size()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7f551af1",
+ "metadata": {},
+ "source": [
+ "은닉 상태 텐서의 크기는 [batch_size, n_tokens, hidden_dim]
\n",
+ "-> 6개의 입력 토큰마다 768차원의 벡터가 반환"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 74,
+ "id": "9a5b950a",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([1, 768])"
+ ]
+ },
+ "execution_count": 74,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "outputs.last_hidden_state[:,0].size()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 75,
+ "id": "c6eb5006",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def extract_hidden_states(batch):\n",
+ " inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}\n",
+ " \n",
+ " # 마지막 은닉 상태를 추출\n",
+ " with torch.no_grad():\n",
+ " last_hidden_state = model(**inputs).last_hidden_state\n",
+ " \n",
+ " # [CLS] 토큰에 대한 벡터를 반환\n",
+ " return {\"hidden_state\": last_hidden_state[:,0].cpu().numpy()}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 77,
+ "id": "1faae4d9",
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "ValueError",
+ "evalue": "PyTorch needs to be installed to be able to return PyTorch tensors.",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[77], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43memotions_encoded\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mset_format\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtorch\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattention_mask\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlabel\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m~/miniforge3/envs/m1_tf_keras/lib/python3.9/site-packages/datasets/dataset_dict.py:548\u001b[0m, in \u001b[0;36mDatasetDict.set_format\u001b[0;34m(self, type, columns, output_all_columns, **format_kwargs)\u001b[0m\n\u001b[1;32m 546\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_values_type()\n\u001b[1;32m 547\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m dataset \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalues():\n\u001b[0;32m--> 548\u001b[0m \u001b[43mdataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mset_format\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_all_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_all_columns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mformat_kwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m~/miniforge3/envs/m1_tf_keras/lib/python3.9/site-packages/datasets/fingerprint.py:480\u001b[0m, in \u001b[0;36mfingerprint_transform.