mirror of
https://github.com/rasbt/LLMs-from-scratch.git
synced 2025-08-10 01:32:10 +00:00
136 lines
24 KiB
Plaintext
136 lines
24 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "a9bc1c1a-53bc-4b86-9140-4f1af0128037",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"<table style=\"width:100%\">\n",
|
||
|
|
"<tr>\n",
|
||
|
|
"<td style=\"vertical-align:middle; text-align:left;\">\n",
|
||
|
|
"<font size=\"2\">\n",
|
||
|
|
"Supplementary code for the <a href=\"http://mng.bz/orYv\">Build a Large Language Model From Scratch</a> book by <a href=\"https://sebastianraschka.com\">Sebastian Raschka</a><br>\n",
|
||
|
|
"<br>Code repository: <a href=\"https://github.com/rasbt/LLMs-from-scratch\">https://github.com/rasbt/LLMs-from-scratch</a>\n",
|
||
|
|
"</font>\n",
|
||
|
|
"</td>\n",
|
||
|
|
"<td style=\"vertical-align:middle; text-align:left;\">\n",
|
||
|
|
"<a href=\"http://mng.bz/orYv\"><img src=\"https://sebastianraschka.com/images/LLMs-from-scratch-images/cover-small.webp\" width=\"100px\"></a>\n",
|
||
|
|
"</td>\n",
|
||
|
|
"</tr>\n",
|
||
|
|
"</table>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "5250207d-f811-46df-9d16-4ac1e9ce1c66",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# Score Correlation Analysis"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "badc7ffb-d51c-4de0-97c5-b54cf3e28315",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"- This notebook analysis the correlation between the different evaluation method scores"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 7,
|
||
|
|
"id": "fa39424b-e058-4351-94ec-249b812ae8fd",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"import json\n",
|
||
|
|
"\n",
|
||
|
|
"with open(\"gpt4-model-1-response.json\", \"r\") as file:\n",
|
||
|
|
" gpt4_model_1 = json.load(file)\n",
|
||
|
|
"\n",
|
||
|
|
"with open(\"gpt4-model-2-response.json\", \"r\") as file:\n",
|
||
|
|
" gpt4_model_2 = json.load(file)\n",
|
||
|
|
"\n",
|
||
|
|
"with open(\"llama3-8b-model-1-response.json\", \"r\") as file:\n",
|
||
|
|
" llama3_8b_model_1 = json.load(file)\n",
|
||
|
|
"\n",
|
||
|
|
"with open(\"llama3-8b-model-2-response.json\", \"r\") as file:\n",
|
||
|
|
" llama3_8b_model_2 = json.load(file)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "4ef67d30-7602-4695-a190-16209a152621",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# GPT-4 vs Llama 3 8B (Model 1)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 8,
|
||
|
|
"id": "2a0d4288-507f-414c-afde-9742935cd8bc",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Correlation coefficient: 0.8048901206421845\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA98ElEQVR4nO3de3gU9cH+/3tz5pBsGpAcJJAVUIigcjYG9VGhUDGtytWWFhSiP1ojqIhHVIxpC8Gnv8daq2BBiVZAqq1YYy3qA1QFA4SjYBTxMRxUklRjsuGQBLLz/SPNliUJbGD2NPt+XVeuNrOfTD6MMLmzM597bIZhGAIAALCoiEBPAAAAwJcIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNKiAj2BYOByufT1118rPj5eNpst0NMBAABeMAxDdXV1SktLU0RE++/fEHYkff3110pPTw/0NAAAwBk4cOCAevbs2e7rhB1J8fHxkpoPVkJCQoBnAwAAvOF0OpWenu7+Od4ewo7kvnSVkJBA2AEAIMSc7hYUblAGAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRoMyAADwiaONTZr3Vpn2fntEGd0666FrM9UpJtLv8wjoOzvvv/++cnJylJaWJpvNptdff93jdcMw9Oijjyo1NVWdOnXS6NGjtWfPHo8x1dXVmjRpkhISEpSYmKhbb71Vhw4d8uOfAgAAnGzan0o14NFVemnDfn2w5xu9tGG/Bjy6StP+VOr3uQQ07Bw+fFgXX3yxnnnmmTZf/+///m899dRTevbZZ7Vx40Z16dJFY8eOVX19vXvMpEmT9PHHH+vdd9/Vm2++qffff1+/+MUv/PVHAAAAJ5n2p1K9W1bV5mvvllX5PfDYDMMw/Pod22Gz2bRy5Updf/31kprf1UlLS9M999yje++9V5JUW1ur5ORkvfDCC5o4caI++eQTZWZmqrS0VMOGDZMkrVq1Stdee62+/PJLpaWlefW9nU6n7Ha7amtreRAoAABn4WhjkwY8uuq04z751bizvqTl7c/voL1Buby8XBUVFRo9erR7m91u18iRI1VSUiJJKikpUWJiojvoSNLo0aMVERGhjRs3trvvhoYGOZ1Ojw8AAHD25r1VZuo4MwRt2KmoqJAkJScne2xPTk52v1ZRUaEePXp4vB4VFaWkpCT3mLYUFhbKbre7P9LT002ePQAA4Wnvt0dMHWeGoA07vjR79mzV1ta6Pw4cOBDoKQEAYAkZ3TqbOs4MQbv0PCUlRZJUWVmp1NRU9/bKykpdcskl7jFVVZ43QB0/flzV1dXur29LbGysYmNjzZ804IUml6FN5dWqqqtXj/g4jXAkKTLCFuhpAUCHNB536aWSvdpXfUS9kzrrpqwMxURF6KFrM/XShv2n/fqHrs30wyybBW3YcTgcSklJ0erVq93hxul0auPGjcrLy5MkZWVlqaamRlu2bNHQoUMlSWvWrJHL5dLIkSMDNXWgXat2HVRBcZkO1v5nRWGqPU75OZkaNzD1FF8JAMGj8K0yLf6gXK4TljjNfesTTbvcodnXZmpMZo92V2NJ0pjMHn7t2wlo2Dl06JA+//xz9+fl5eXavn27kpKS1KtXL82cOVO/+c1v1K9fPzkcDs2ZM0dpaWnuFVsDBgzQuHHjNG3aND377LM6duyYZsyYoYkTJ3q9Egvwl1W7Dipv6VadvPyxorZeeUu3auHkIQQeAEGv8K0y/fH98lbbXYbc208VdLx53WwBvWdn8+bNGjx4sAYPHixJmjVrlgYPHqxHH31UknT//ffrjjvu0C9+8QsNHz5chw4d0qpVqxQXF+fex7Jly9S/f39dc801uvbaazVq1CgtWrQoIH8eoD1NLkMFxWWtgo4k97aC4jI1uYKiCQIA2tR43KXFH7QOOidqKwi1ZcNn35oxJa8ETc9OINGzA18r+b9v9bPFG0477uVplyqrTzc/zAgAOu75D77Qr//+iWn72zt//Fl9fcj37ABWUlVXf/pBHRgHAIGwr9p/y8XNRNgB/KBHfNzpB3VgHAAEQu8k/y0XNxNhB/CDEY4kpdrj1N4Cc5uaV2WNcCT5c1oA0CE3ZWXIrKaMnw9vvyLGbIQdwA8iI2zKz2nulDj5PNHyeX5OJn07AIJaTFSEpl3uOOWYzNR4r/YVFeW/vjvCDuAn4wamauHkIUqxe16qSrHHsewcQMiYfW2mfnmFo9U7PBE26ZdXODRhSE+v9uPPS2KsxhKrseBfNCgDsIL2GpT/5WzQ8Hn/e9qvL31otM5JOLt3d7z9+U3YEWEHAACzTFiwXlv215x23NBeifrr7dln9b1Yeg4AAPzu61rvKjS8HWcGwg4AADCNPdq7C0bejjMDYQcAAJjm8+oGU8eZgbADAABMc9xl7jgzEHYAAIBpor1MFt6OMwNhBwAAmObNGVeYOs4MhB0AAGCa6qONpo4zA2EHAACYpqrOuyXl3o4zA2EHAACYpkd83OkHdWCcGaL89p0AwM94NAfgfyMcSUq1x6mitl5tNenY1PxMwBGOJL/NibADwJJW7TqoguIyHTyhpTXVHqf8nEweugr4UGSETfk5mbpt6dY2Xzck5edk+vUXDy5jAbCcVbsOKm/pVo+gI0kVtfXKW7pVq3YdDNDMgPCw4J+fn9XrZiPsALCUJpehguKyNt8+b9lWUFymJlfYPwMZ8IlD9cf10ZfOU4756EunDtUf99OMCDsALGZTeXWrd3ROZEg6WFuvTeXV/psUEEbu/vM2U8eZgbADwFKCcdkrEE72f3fU1HFmIOwAsJRgXPYKhJNe3+tk6jgzEHYAWErLstf21nnY1Lwqy5/LXoFwYBiGPvy/b9Tk5QM+f/fTwb6d0AlYeg7AUlqWveYt3Sqb5HGjcksA8veyV8DK6o816Y3tX2vJ+nJ9WlHn1ddc1DNBXeP8F0EIOwAsZ9zAVC2cPKRVz04KPTuAaSqd9XqpZJ+Wb9qv6sPNz7nqFB2pG4ecq9e3faXDjU3tfu3eb474a5qSCDsALGrcwFSNyUyhQRkw2fYDNSpaX66/f3RQx/9d4XBuYifdnNVbE4f3UuNxl5Zt3H/KfTjrj+tfzgadkxDrjykTdgBYV2SETVl9ugV6GkDIO9bk0qpdFSpaX66t+2vc24dnfE+52Q59PzNZUZHNtwGPmr/aq33esGCd1j14jS+m2wphBwAAtOm7w416uXS/XirZ574kHB1pU85FacrNdmhQT3urr6k+fMyrfXs7zgyEHQAA4OGzyjoVrS/Xym1fqf5Y8/Kq7l1jNGlkb026tNcpqxuSukTrSE379+ucOM5fCDsAAEAul6G1u6tUtH6v1n3+jXv7hWkJys12KOfiVMVGRZ52PytvH6Xh8/7Xq3H+QtgBACCMHWo4rr9sPqAXS/ap/JvDkqQIm/T9zBTlZmdohCNJNpv3N/afkxCrhLgoOU/x7KuEuCi/3ZwsEXYAAAhL+789ohdL9uqV0gOqa2gOJvFxUZo4PF03Z2UoPanzGe/7o8fG6qLH3m4z8CTERemjx8ae8b7PBGHHR5pcBkteAQBBxTAMbfiiWkvWl+t/P6mU8e/WzfO6d9HU7AxNGNJTXWLNiQYfPTZW/3I26IYF61R9+JiSukRr5e2j/PqOTgvCjg+s2nWwVZlZKmVmAIAAaa/l+Irzz1Fudoau7HeOInzwC/k5CbF+W15+KoQdk63adVB5S7d6VNRLUkVtvfKWbtXCyUMIPAAAv6h01mvphn1avnG/vj2p5Tg3O0N9e8QHeIb+QdgxUZPLUEFxWaugIzU/n8cmqaC4TGMyU7ikBQDwmR3/bjl+84SW4zR7nG6+LEMTh6crsXNMgGfoX4QdE20qr/a4dHUyQ9LB2nptKq+m1RUAYKpjTS69/XGFlqw7fctxuCHsmKiqrv2gcybjAAA4nTNpOQ43hB0TnapR8kzGAQDQnuaW471aue1Lj5bjn4/srcmnaTkON4QdE41wJCnVHqeK2vo279uxSUqxNy9DhzlY4g8gnLhchv75WZW
|
||
|
|
"text/plain": [
|
||
|
|
"<Figure size 640x480 with 1 Axes>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "display_data"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import matplotlib.pyplot as plt\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"list1, list2 = gpt4_model_1, llama3_8b_model_1\n",
|
||
|
|
"correlation_matrix = np.corrcoef(list1, list2)\n",
|
||
|
|
"correlation = correlation_matrix[0, 1]\n",
|
||
|
|
"print(f\"Correlation coefficient: {correlation}\")\n",
|
||
|
|
"\n",
|
||
|
|
"plt.scatter(list1, list2)\n",
|
||
|
|
"plt.plot(\n",
|
||
|
|
" np.unique(list1),\n",
|
||
|
|
" np.poly1d(np.polyfit(list1, list2, 1))(np.unique(list1))\n",
|
||
|
|
")\n",
|
||
|
|
"plt.xlabel(\"GPT-4\")\n",
|
||
|
|
"plt.ylabel(\"Llama3 8B\")\n",
|
||
|
|
"plt.show()"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3 (ipykernel)",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.11.4"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 5
|
||
|
|
}
|