From 77c563f358741d295ca2e7e6241bbae9fd26dc87 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 28 Aug 2025 16:46:24 +0000 Subject: [PATCH] instruct mode added to pipiline --- .../requirements-checkpoint.txt | 1 + .ipynb_checkpoints/setup-checkpoint.sh | 46 + .../.ipynb_checkpoints/sample-checkpoint.yaml | 6 +- configs/instruct/sample.yaml | 6 +- ..._(14B)_Conversational (2)-checkpoint.ipynb | 8668 +++++++++++++++++ ...en2_5_Coder_(14B)_Conversational (2).ipynb | 8668 +++++++++++++++++ .../data_processor-checkpoint.py | 913 ++ .../inference-checkpoint.py | 393 + .../.ipynb_checkpoints/train-checkpoint.py | 518 + pipelines/instruct/data_processor.py | 128 +- pipelines/instruct/train.py | 171 +- requirements.txt | 1 + setup.sh | 46 + utils/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 148 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 155 bytes .../config_manager.cpython-310.pyc | Bin 0 -> 4798 bytes 16 files changed, 19404 insertions(+), 161 deletions(-) create mode 100644 .ipynb_checkpoints/setup-checkpoint.sh create mode 100644 notebooks/.ipynb_checkpoints/Qwen2_5_Coder_(14B)_Conversational (2)-checkpoint.ipynb create mode 100644 notebooks/Qwen2_5_Coder_(14B)_Conversational (2).ipynb create mode 100644 setup.sh create mode 100644 utils/__pycache__/__init__.cpython-310.pyc create mode 100644 utils/config/__pycache__/__init__.cpython-310.pyc create mode 100644 utils/config/__pycache__/config_manager.cpython-310.pyc diff --git a/.ipynb_checkpoints/requirements-checkpoint.txt b/.ipynb_checkpoints/requirements-checkpoint.txt index 862bfea..5346e99 100644 --- a/.ipynb_checkpoints/requirements-checkpoint.txt +++ b/.ipynb_checkpoints/requirements-checkpoint.txt @@ -9,3 +9,4 @@ numpy>=1.24.0 scikit-learn>=1.3.0 pyyaml>=6.0 huggingface-hub>=0.15.0 +unsloth diff --git a/.ipynb_checkpoints/setup-checkpoint.sh b/.ipynb_checkpoints/setup-checkpoint.sh new file mode 100644 index 0000000..51b0baa --- /dev/null +++ b/.ipynb_checkpoints/setup-checkpoint.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -e # exit on error + +# Step 1: Download Miniconda installer +echo ">>> Downloading Miniconda..." +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh + +# Step 2: Run the installer +echo ">>> Installing Miniconda..." +bash ~/miniconda.sh -b -p $HOME/miniconda + +# Step 3: Initialize conda for bash +echo ">>> Initializing Conda..." +eval "$($HOME/miniconda/bin/conda shell.bash hook)" + +# Step 4: Add conda to PATH permanently +if ! grep -q "miniconda/bin" ~/.bashrc; then + echo "export PATH=\$HOME/miniconda/bin:\$PATH" >> ~/.bashrc +fi +source ~/.bashrc + +# Step 5: Verify installation +echo ">>> Conda version:" +conda --version + +# Step 6: Create environment +echo ">>> Creating environment: llm-env (python=3.10)..." +conda create -n llm-env python=3.10 -y + +# Step 7: Activate environment +echo ">>> Activating environment..." +conda activate llm-env + +# Step 8: Install dependencies +echo ">>> Installing scikit-learn..." +conda install scikit-learn -y + +if [ -f requirements.txt ]; then + echo ">>> Installing Python requirements..." + pip install -r requirements.txt +else + echo ">>> No requirements.txt found, skipping pip install." +fi + +echo ">>> Setup complete. To activate your environment run:" +echo "conda activate llm-env" diff --git a/configs/instruct/.ipynb_checkpoints/sample-checkpoint.yaml b/configs/instruct/.ipynb_checkpoints/sample-checkpoint.yaml index 17fc405..b6fe30a 100644 --- a/configs/instruct/.ipynb_checkpoints/sample-checkpoint.yaml +++ b/configs/instruct/.ipynb_checkpoints/sample-checkpoint.yaml @@ -10,7 +10,7 @@ task: # Data Processing Configuration data: source: "custom" # Data source: "huggingface" or "custom" - data_path: "./data/raw/instruct/code_reasoning.jsonl" # Path to conversation data file + data_path: "data/raw/swe_reasoning_dataset (3).jsonl" # Path to conversation data file data_format: "jsonl" # Data format: "jsonl", "json" # Field Mapping for Conversation Data @@ -34,7 +34,7 @@ data: # Model Configuration model: - name: "unsloth/Qwen2.5-72B-Instruct" # Model name from HuggingFace Hub (optimized for instruction following) + name: "unsloth/Qwen2.5-Coder-7B" # Model name from HuggingFace Hub (optimized for instruction following) max_length: 2048 # Maximum sequence length for tokenization max_seq_length: 2048 # Maximum sequence length for training (RoPE scaling supported) dtype: null # Data type: null for auto detection, float16 for Tesla T4/V100, bfloat16 for Ampere+ @@ -42,7 +42,7 @@ model: token: null # HuggingFace token for gated models (e.g., "hf_...") # Training Model Parameters - training_model: "unsloth/Qwen2.5-72B-Instruct" # Model to use for training + training_model: "unsloth/Qwen2.5-Coder-7B" # Model to use for training training_max_seq_length: 2048 # Max sequence length for training training_dtype: null # Data type for training training_load_in_4bit: true # 4bit quantization for training diff --git a/configs/instruct/sample.yaml b/configs/instruct/sample.yaml index 17fc405..b6fe30a 100644 --- a/configs/instruct/sample.yaml +++ b/configs/instruct/sample.yaml @@ -10,7 +10,7 @@ task: # Data Processing Configuration data: source: "custom" # Data source: "huggingface" or "custom" - data_path: "./data/raw/instruct/code_reasoning.jsonl" # Path to conversation data file + data_path: "data/raw/swe_reasoning_dataset (3).jsonl" # Path to conversation data file data_format: "jsonl" # Data format: "jsonl", "json" # Field Mapping for Conversation Data @@ -34,7 +34,7 @@ data: # Model Configuration model: - name: "unsloth/Qwen2.5-72B-Instruct" # Model name from HuggingFace Hub (optimized for instruction following) + name: "unsloth/Qwen2.5-Coder-7B" # Model name from HuggingFace Hub (optimized for instruction following) max_length: 2048 # Maximum sequence length for tokenization max_seq_length: 2048 # Maximum sequence length for training (RoPE scaling supported) dtype: null # Data type: null for auto detection, float16 for Tesla T4/V100, bfloat16 for Ampere+ @@ -42,7 +42,7 @@ model: token: null # HuggingFace token for gated models (e.g., "hf_...") # Training Model Parameters - training_model: "unsloth/Qwen2.5-72B-Instruct" # Model to use for training + training_model: "unsloth/Qwen2.5-Coder-7B" # Model to use for training training_max_seq_length: 2048 # Max sequence length for training training_dtype: null # Data type for training training_load_in_4bit: true # 4bit quantization for training diff --git a/notebooks/.ipynb_checkpoints/Qwen2_5_Coder_(14B)_Conversational (2)-checkpoint.ipynb b/notebooks/.ipynb_checkpoints/Qwen2_5_Coder_(14B)_Conversational (2)-checkpoint.ipynb new file mode 100644 index 0000000..03bcf8d --- /dev/null +++ b/notebooks/.ipynb_checkpoints/Qwen2_5_Coder_(14B)_Conversational (2)-checkpoint.ipynb @@ -0,0 +1,8668 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "1UdZgdMa6L9X" + }, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xieWpvmi6L9Z" + }, + "source": [ + "### News" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T5zA3wvi6L9Z" + }, + "source": [ + "### Installation" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "yxLJvyfz6L9Z" + }, + "outputs": [], + "source": [ + "%%capture\n", + "import os, re\n", + "if \"COLAB_\" not in \"\".join(os.environ.keys()):\n", + " !pip install unsloth\n", + "else:\n", + " # Do this only in Colab notebooks! Otherwise use pip install unsloth\n", + " import torch; v = re.match(r\"[0-9\\.]{3,}\", str(torch.__version__)).group(0)\n", + " xformers = \"xformers==\" + (\"0.0.32.post2\" if v == \"2.8.0\" else \"0.0.29.post3\")\n", + " !pip install --no-deps bitsandbytes accelerate {xformers} peft trl triton cut_cross_entropy\n", + " !pip install sentencepiece protobuf \"datasets>=3.4.1,<4.0.0\" \"huggingface_hub>=0.34.0\" hf_transfer\n", + " !pip install \"unsloth_zoo[base] @ git+https://github.com/unslothai/unsloth-zoo\"\n", + " !pip install \"unsloth[base] @ git+https://github.com/unslothai/unsloth\"\n", + "!pip install transformers==4.55.4" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hSZDfEpp6L9a" + }, + "source": [ + "### Unsloth" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 880, + "referenced_widgets": [ + "298be59fbb97424080819b7a16db2024", + "b851f3f552ad41fe97e9d7b72ea71b1c", + "963f33764e744624bd33a8252511c0f2", + "5f9574f2a91742d6a11be231ae60b4a6", + "895dccba65a5482c94d4c74371412241", + "39c1e706d28a46218940e01190b2843a", + "5d97ec03ac9544409f51c2ae1759dc8b", + "7135057d6cb84a77a04a8db3c6c3f336", + "a12e1cf419bd446b9823fc924431cae4", + "035a4fa66ce8447fbbed4adfd986726b", + "16dee8a92a9e4fbd908eeda7193fc190", + "da6b6fcb41f74c73b0b3460ba6d4c7dd", + "9e7f78f4d932445daecd09dcf7b37859", + "7942f894ea8f4268b9c2b6ad83b2f279", + "dfed5a49746e413bbaa44aa82a61228c", + "6db5bed4b1c4478c97e07a7ab32d1845", + "994b86b8732346de8b0789f366068549", + "3b74d36ab9974076b06ffe7b545d582f", + "9c40f79c7efb4b758c6af85ba105fe4c", + "413290be48d2410c93e2da6848affa39", + "1105390239d846898e653a7bfd092ee6", + "9b816b35709d4b4fbedd183e6a7aae5a", + "6b81f7175b224cb4a4320b33d969c146", + "41c627c3ffb144eca3cd45f490e61a30", + "6178ea883ce546fd9f433d5a683d2d18", + "260322195cc9480aaeb1173315a93894", + "c69c5e05efc8464f86c662b9ae931538", + "d67e9d729bee4325a5a01a90aecb52d0", + "7756a42c820745389107246cb6a35072", + "99c57c528f2d4c6284022efba3b9e81d", + "1ccf810e39ea4fbcb81f465c3b6bd601", + "55f6a6b67a754ef89732a29531dcf34a", + "25d1aaf77b304450877876f5c8bd38fb", + "3032a337e20945198e3678f91c371bc9", + "e1d1a5a6b94741a4b22e254c504264c0", + "63b67e250d8349f88ee276ef732168c9", + "cae54a63540742c3bb7a0412a5c55add", + "ae9f5ce0eefc4a8a98c2599a0fbf890a", + "d71c0ead75c2411587677493ad59256b", + "c186068d654b4ba3a97105956d52083e", + "0f65529309064ff781015f38375a34ce", + "7350d931506e4e99b55a03751c453515", + "04c78130763c4dfc93a86c788a6e01e8", + "e3a4437a1f8b4710a77983b82f0ebe30", + "c9c96e91dc6347e1b6566570e925e40f", + "2853c5ef931d422aa5a7622bc2f61bb2", + "8892267f311b4cf0a5267ce193c5a646", + "789512128fa84d3481a086db62c44c37", + "8f9648d8465c414ea4699a63fbf936a9", + "f15f91be9d1e47da8079d1d0d48f7842", + "17f32b453e4841a383c210853a36da84", + "fa44d5de08014ac48013167025a3de1b", + "c9f72c0ebbee4aad8023b2082ce59f34", + "e658cae26b3f457882ef5fe90309ed38", + "4e38a0d4d6ca440388951805ce4b6104", + "4949d26418b944f0a7d4c8e69a49f3f7", + "edf75f353d9645ba8796e2652d636166", + "b6c160bf3d3043f8840ab4b7431301ca", + "cacb47001cd341abae219b526401616d", + "1c971ddb47764c6aa4f766c5abd8cd71", + "76a9c48d89e44cfc9d30cdf3b56a4f74", + "2983c2011c194d0b88383d3bbf356845", + "8dd6e110704f40c995b9f0f6951cbb74", + "c2dd2c13d62d48e09a68373f10dcd95a", + "97359398bd724bf1acf5c4370bb7eaa2", + "06ad2446cdb64c4db134046da4cfd0a5", + "815aec1656224eacabc076618f7f067a", + "4d78f66177ca43f6b1efa5588078c0e4", + "0526bd2f6b014475bc4279fb6ad44ab0", + "55aec8e495624b2faa5492e612807dbb", + "e6eccabf31e545a2b06c7650c9781d05", + "553a899890fb42348e5447ed43d5fd9b", + "b985370c39e64ea0943e1edda65a6bed", + "1a89be1d70a043a7ab316fe731a4af95", + "cfd8bbe5291845f6b886415ece13f7f7", + "2cd72f4bdc1b41a4a2894c553d335dc5", + "9e31bce1bd2c4cf48724f9f5d9c5282f", + "f6bee8f11d694040ae78e6f8616070d0", + "f7646ab3a3ff4106b33ab158c0296ae7", + "3c58773e281144f39081dc3414f9f4c9", + "7972d106d5414a218441942913aa3e3c", + "b1a3ff316c374f509898c21c8618dcc9", + "bbac5526b00745c982a175bd180cbd24", + "5d053c12d1ae427fa34c553d4759e25d", + "71b65997837d405ebc3e92cb8b498faa", + "bf4e7a04363d477cb36586a0fbcfc64e", + "167f32a4f26a4466a3c9b94fb7aa34b2", + "d5682db4b5cd4bb4abdb6ff7258caefa", + "ec4c9d4fe69645d78243987a8c12d503", + "734f1dd7724d49f5a196552da6abf192", + "fcea53819cc441ec907174ee79ccc8ee", + "5f63b5e75daf44ada6af8dd6922a58ce", + "31400dbf077840769de1cb7f63cec131", + "b00ce2847b33492b9eba44d065ea7f06", + "7ba890d72f4245cb8f760f252ff80caa", + "d6d14db2e4894ad18b899cafb148b871", + "f93b699993824be7a005d54d96cd6361", + "b2def88c44ee4b608faaf38148b30c18", + "08d6a0314e864384aa3e163fb2224c35", + "b718bba8164047ccbb046d3475fb0f12", + "25ce0fb1aed24b0cb1419ac3761109f7", + "b468a2d515f943979af3d1921cda5594", + "89520764d8ef4805805055bebd21273e", + "adc2a364ac1b4901acbae39562402743", + "9d15aa6b077044f4a7aba7c3d543c984", + "9a2c98e1c8d5491791e4d7a1390cf81f", + "e7cf601a9b6342cfb38376989e08c5a9", + "ca23b61a88bf4e609075619b9d1111ef", + "56146e26a4054d87a843ace9758b7100", + "02d6ff36400543d196671dcb609a3554", + "5998b125c6a64ef886fc51d745b5301d", + "e2a1b3b26e84490c99b305d4a651cda4", + "5341de66236b459ca83a154fb65e0596", + "0dfde06ab9d54e6b8c563d380f4b6026", + "27f38f1ed0a14f119ba6b8f48662f5b9", + "7b2494ece7db465ca088670a7f8ffbef", + "03ea102e7b20453b939c9e514f7a61d5", + "0dd1d1bffd654340b8d21af1f0eab11d", + "f16ef7e24cd64841a3e2d8988d286c58", + "c4df1f0a6cb34bca9a84cb734ffcd6c7", + "8a87d368abe3434c82f99fb23e717e5e", + "213d400d9c704b9aaed21ad5a91c91a7", + "d09a3f6c289c4608a8dfcf53157bf1b6", + "1ad8199be4dd49ea8a9638d3e53d3d74", + "071495bad0e143ce8b5514d75ce5d435", + "7d7b05e1f1f642acb8f5ba6afdf6d82f", + "019f19eb060545539da7628564caaf5e", + "3973d4960a584bcebe080796c99104f1", + "8409a21d6d594569aca7721796af52ba", + "c883d3a00bc542db88244a71b924e1f0", + "9dd47b04490548bbb03c204544cc022e", + "524a6d91d0924912b4ce4a2f4c87e76a" + ] + }, + "id": "QmUBVEnvCDJv", + "outputId": "833593bc-378f-4726-ac6f-8bf57b0ee091" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "🦥 Unsloth Zoo will now patch everything to make training faster!\n", + "==((====))== Unsloth 2025.8.9: Fast Qwen2 patching. Transformers: 4.55.4.\n", + " \\\\ /| NVIDIA A100-SXM4-80GB. Num GPUs = 1. Max memory: 79.254 GB. Platform: Linux.\n", + "O^O/ \\_/ \\ Torch: 2.8.0+cu128. CUDA: 8.0. CUDA Toolkit: 12.8. Triton: 3.4.0\n", + "\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.32.post2. FA2 = False]\n", + " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", + "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Unsloth: unsloth/qwen2.5-72b-instruct-bnb-4bit can only handle sequence lengths of at most 32768.\n", + "But with kaiokendev's RoPE scaling of 1.465, it can be magically be extended to 48000!\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "dbc5546abfd948f19843459c132922f8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/9 [00:00 0 ! Suggested 8, 16, 32, 64, 128\n", + " target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", + " \"gate_proj\", \"up_proj\", \"down_proj\",],\n", + " lora_alpha = 16,\n", + " lora_dropout = 0, # Supports any, but = 0 is optimized\n", + " bias = \"none\", # Supports any, but = \"none\" is optimized\n", + " # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n", + " use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n", + " random_state = 3407,\n", + " use_rslora = False, # We support rank stabilized LoRA\n", + " loftq_config = None, # And LoftQ\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vITh0KVJ10qX" + }, + "source": [ + "\n", + "### Data Prep\n", + "We now use the `Qwen-2.5` format for conversation style finetunes. We use [Maxime Labonne's FineTome-100k](https://huggingface.co/datasets/mlabonne/FineTome-100k) dataset in ShareGPT style. But we convert it to HuggingFace's normal multiturn format `(\"role\", \"content\")` instead of `(\"from\", \"value\")`/ Qwen renders multi turn conversations like below:\n", + "\n", + "```\n", + "<|im_start|>system\n", + "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", + "<|im_start|>user\n", + "What is 2+2?<|im_end|>\n", + "<|im_start|>assistant\n", + "It's 4.<|im_end|>\n", + "\n", + "```\n", + "\n", + "We use our `get_chat_template` function to get the correct chat template. We support `zephyr, chatml, mistral, llama, alpaca, vicuna, vicuna_old, phi3, llama3` and more." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 10000 examples\n", + "Dataset({\n", + " features: ['conversation', 'metadata', 'original_example'],\n", + " num_rows: 10000\n", + "})\n", + "{'conversation': [{'content': 'You are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.', 'role': 'system'}, {'content': 'I\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\nElasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n\\n\\nCan you help me debug this systematically and explain your reasoning?', 'role': 'user'}, {'content': '', 'role': 'assistant'}], 'metadata': {'base_commit': '160cfef6e1118061fa66d333e8c2a572f5d0a815', 'dataset': 'princeton-nlp/SWE-bench', 'generation_timestamp': '2025-08-19T23:35:38.773997', 'has_patch': True, 'instance_id': 'DataDog__integrations-core-10093', 'original_index': 0, 'reasoning_length': 0, 'repo': 'DataDog/integrations-core', 'source': 'swe-bench'}, 'original_example': {'instance_id': 'DataDog__integrations-core-10093', 'patch': \"diff --git a/elastic/datadog_checks/elastic/elastic.py b/elastic/datadog_checks/elastic/elastic.py\\n--- a/elastic/datadog_checks/elastic/elastic.py\\n+++ b/elastic/datadog_checks/elastic/elastic.py\\n@@ -139,11 +139,17 @@ def _get_es_version(self):\\n try:\\n data = self._get_data(self._config.url, send_sc=False)\\n raw_version = data['version']['number']\\n+\\n self.set_metadata('version', raw_version)\\n # pre-release versions of elasticearch are suffixed with -rcX etc..\\n # peel that off so that the map below doesn't error out\\n raw_version = raw_version.split('-')[0]\\n version = [int(p) for p in raw_version.split('.')[0:3]]\\n+ if data['version'].get('distribution', '') == 'opensearch':\\n+ # Opensearch API is backwards compatible with ES 7.10.0\\n+ # https://opensearch.org/faq\\n+ self.log.debug('OpenSearch version %s detected', version)\\n+ version = [7, 10, 0]\\n except AuthenticationError:\\n raise\\n except Exception as e:\\n\", 'problem_statement': 'Elasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n', 'repo': 'DataDog/integrations-core'}}\n" + ] + } + ], + "source": [ + "from datasets import Dataset\n", + "import json\n", + "\n", + "file_path = \"data/swe_reasoning_dataset (3).jsonl\"\n", + "\n", + "# Load JSONL\n", + "data = []\n", + "with open(file_path, \"r\", encoding=\"utf-8\") as f:\n", + " for line in f:\n", + " data.append(json.loads(line))\n", + "\n", + "print(f\"Loaded {len(data)} examples\")\n", + "\n", + "# Convert to HuggingFace Dataset\n", + "dataset = Dataset.from_list(data)\n", + "\n", + "print(dataset)\n", + "print(dataset[0]) # Show first example\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 141, + "referenced_widgets": [ + "e7371eff6f4c458792abd1e05e147110", + "b27f191668cf4d1cbf9705fe51e3c3df", + "c19fa4dea6524d37843cbf5786b26221", + "594a5e73c6d24e8a8f98400c2ee99636", + "709b591f878747ee9a22c01bb8fa2c74", + "6e2018997fc545d8bb37c862fff0c751", + "acda49009d124138843b187c2aa9e410", + "fba4df061ec84f4abe521ee628c53b62", + "e0cf51730c35468ea3074be96df8b8a0", + "38db9cffac4a42759c3c1be2e5a4dd3f", + "32b193ccdb024d5993e42b4e0efd1631", + "f4a36fec7d7a41e49bf125cc5cc98b0f", + "a2951310cc79437a8dac88cadd9ed1e6", + "8cf481de1931406e97951a2adfbdeee0", + "24f12af7e8664d72ae38b06a713294a5", + "0e66ab8fb2384f09a303a034fbde6247", + "254584c067e146809ab71a7a85d7c7a4", + "44fdce605bf54fba80f81524461259bf", + "101d6d98e8bb4a1ab216ddac084314ec", + "617973da3a3f4b1f86729365d12f9875", + "a7d2b0d1a685407fa0633eb56f3676db", + "7dbc6c270eff43569640b25e450601ab", + "6765cb1a549f45e998ccb19139e2e8a3", + "c2d9520cf1134cc182c2612bfd912875", + "a53fab9c72f6468aaffd5bab3765af49", + "244e4cf841cd483b8e9e165c77c471f0", + "8bbeb59a93de469682466e54bc6c35a7", + "0df17c7f00c949c7b8444b91754fa4dd", + "a8f4a39bb9d74a9bb4138c8bed4186e5", + "8fe160b637b24a74a4a97bdd9b1858d1", + "57c9f679f77241d0b6f9bf794dad2b9b", + "b47587f1ba944ad9850fcf555cdbcf12", + "82806a720dad4704ba224812d9018bc2" + ] + }, + "id": "LjY75GoYUCB8", + "outputId": "5cf90520-4c68-4844-c050-b72a075f3ee6" + }, + "outputs": [], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template = \"qwen-2.5\",\n", + ")\n", + "\n", + "def formatting_prompts_func(examples):\n", + " convos = examples[\"conversation\"]\n", + " texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for convo in convos]\n", + " return { \"text\" : texts, }\n", + "pass\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K9CBpiISFa6C" + }, + "source": [ + "We now use `standardize_sharegpt` to convert ShareGPT style datasets into HuggingFace's generic format. This changes the dataset from looking like:\n", + "```\n", + "{\"from\": \"system\", \"value\": \"You are an assistant\"}\n", + "{\"from\": \"human\", \"value\": \"What is 2+2?\"}\n", + "{\"from\": \"gpt\", \"value\": \"It's 4.\"}\n", + "```\n", + "to\n", + "```\n", + "{\"role\": \"system\", \"content\": \"You are an assistant\"}\n", + "{\"role\": \"user\", \"content\": \"What is 2+2?\"}\n", + "{\"role\": \"assistant\", \"content\": \"It's 4.\"}\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 157, + "referenced_widgets": [ + "dee9866d465e43c99e61f71b819d9994", + "ba0b2bd35ce44100be27bc749642e1cc", + "b83dcf4897f544678d85a072cfc8bcca", + "3215fa6f5c0446b9b6b4a0d15dd33032", + "79ce9732a3af4920a64598318a491398", + "16946595a9064832a233c889e01818c9", + "ebdb9c8af7be4743b2e3c1308bddc0a5", + "a6baf9fd477342fe8474022ac8e7a6ed", + "3371554f74674d2da66fc8c794822c24", + "bccfa4e6adfd40d3afff51f5ab038422", + "d6292ba7c64e4e3f9baf68905657ec56", + "ffcb4731dff04faa8323dceae528bce3", + "8bcfc8c1bd6c474fabac2dfff5ca829d", + "b608a884c04541ca88253a72d1d81cb6", + "975b0754e56546d4bd7103b4f28a37c5", + "3ae2a86a77a34f61a204bd1355dda8b3", + "23cf64d2dff94432af9cf6a7f99744ae", + "bfef7ae67a7b43ed816246be0ed19a79", + "760c42f3b2054820acf6b1b6da11356a", + "cf74cf0282cd409498112783ef3068aa", + "e38e2b0f246d4b369487fd405be38e15", + "6b4a5dee5e3143a9b69dede345b7187d" + ] + }, + "id": "oPXzJZzHEgXe", + "outputId": "a1d12c39-303d-4ffc-83e9-3e4dd35cf172" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1110cc5449b44a60a688b90331bd35ad", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map: 0%| | 0/10000 [00:00system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n<|im_start|>user\\nHow do astronomers determine the original wavelength of light emitted by a celestial body at rest, which is necessary for measuring its speed using the Doppler effect?<|im_end|>\\n<|im_start|>assistant\\nAstronomers make use of the unique spectral fingerprints of elements found in stars. These elements emit and absorb light at specific, known wavelengths, forming an absorption spectrum. By analyzing the light received from distant stars and comparing it to the laboratory-measured spectra of these elements, astronomers can identify the shifts in these wavelengths due to the Doppler effect. The observed shift tells them the extent to which the light has been redshifted or blueshifted, thereby allowing them to calculate the speed of the star along the line of sight relative to Earth.<|im_end|>\\n'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset[5][\"text\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "idAEIeSQ3xdS" + }, + "source": [ + "\n", + "### Train the model\n", + "Now let's train our model. We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support TRL's `DPOTrainer`!\n", + "\n", + "The trainer includes our **gradient accumulation bug fix**. Read more about it here: [Blog post](https://unsloth.ai/blog/gradient)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 67, + "referenced_widgets": [ + "93ccb0dfad4e4d60a7e7f42402bff80d", + "8c6346ded74b4614871353d09b685ab3", + "7a285248d8fa4378860cd5e899613057", + "8b1c036ade754137a675d55d6ad12e2c", + "b9304f02c4774b6eb161da6fa6b009ff", + "7cbc2991535b440b97e02a9df10b306a", + "ac9c54ff279a4080b7485d669989db6d", + "ba5b4ef26bc14dadbd444c7a039a2995", + "18a1a798b3824b33b73f345520583d91", + "6cbf4a502296486d9b0d8588dc123732", + "7382f0c83e7b44d3a19bbfda0d3723ee" + ] + }, + "id": "95_Nn-89DhsL", + "outputId": "316cba0c-e677-47d5-c678-5b8398facc1e" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8a541f2dcfe840438d4c61f846417970", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Unsloth: Tokenizing [\"text\"] (num_proc=2): 0%| | 0/10000 [00:00user\\n\",\n", + " response_part = \"<|im_start|>assistant\\n\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Dv1NBUozV78l" + }, + "source": [ + "We verify masking is actually done:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 175 + }, + "id": "LtsMVtlkUhja", + "outputId": "cadeca7a-43d9-4a4b-bc80-c25cff082e2e" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'<|im_start|>system\\nYou are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.<|im_end|>\\n<|im_start|>user\\nI\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\n[apache] fix metric collection\\n### What does this PR do?\\r\\n\\r\\nThis collects the correct data for `apache.net.bytes_per_s` and `apache.net.request_per_s`.\\r\\n\\r\\nhttps://www.datadoghq.com/blog/collect-apache-performance-metrics\\r\\nhttps://wiki.opennms.org/wiki/Monitoring_Apache_with_the_HTTP_collector\\r\\nhttps://httpd.apache.org/docs/current/mod/mod_status.html\\r\\n\\r\\n### Motivation\\r\\n\\r\\nCustomer encountered bug\\n\\n\\nCan you help me debug this systematically and explain your reasoning?<|im_end|>\\n<|im_start|>assistant\\n## INITIAL PROBLEM ASSESSMENT\\n\\nThe bug report indicates that there was an issue with metric collection for Apache performance metrics, specifically `apache.net.bytes_per_s` and `apache.net.request_per_s`. This suggests that the application was not reporting accurate or expected values for these metrics, which could lead to misinterpretation of server performance by users relying on this data.\\n\\nSymptoms might include:\\n- Users not seeing expected values in monitoring dashboards.\\n- Discrepancies in data reported over time, potentially leading to false conclusions about server load and performance.\\n- Customer complaints about the monitoring system\\'s reliability.\\n\\n## HYPOTHESIS FORMATION\\n\\n1. **Incorrect Metric Mapping**: The metrics `apache.net.bytes_per_s` and `apache.net.request_per_s` might not be correctly mapped in the code, leading to values not being collected or reported as intended.\\n \\n2. **Improper Metric Calculation**: There may be a logical error in the way metrics are calculated or aggregated, such as not accounting for rates correctly.\\n\\n3. **Configuration Issues**: There could be an issue with how Apache is configured, potentially affecting what metrics are exposed and collected by the monitoring tool.\\n\\n**Ranked by likelihood**: \\n1. Incorrect Metric Mapping \\n2. Improper Metric Calculation \\n3. Configuration Issues\\n\\n## SYSTEMATIC INVESTIGATION\\n\\n1. **Review Code for Metric Mapping**:\\n - Examine the sections of code where metrics are defined and mapped to ensure that `apache.net.bytes_per_s` and `apache.net.request_per_s` are correctly included in the collection logic.\\n - Look for any recent changes in the code that might have affected the mapping.\\n\\n2. **Check Metric Calculation Logic**:\\n - Investigate how the metrics are being calculated. Look for functions or methods that aggregate or compute the values for these metrics.\\n - Confirm if the metrics are being treated as rates (per second) and check the logic for calculating rates.\\n\\n3. **Analyze Apache Configuration**:\\n - Validate the Apache configuration to ensure that the necessary modules (like `mod_status`) are enabled and that the URLs used for metric collection are correct.\\n - Check if the Apache server is correctly exposing the expected data needed for the metrics.\\n\\n## ROOT CAUSE IDENTIFICATION\\n\\nUpon reviewing the code changes in the provided patch, it becomes evident that the root cause was related to the way metrics were structured and collected. The initial code had the metrics defined in one section but did not correctly leverage them in the main collection logic. The patch introduced a dedicated section for rates (`RATES`), clearly defining how to collect `apache.net.bytes_per_s` and `apache.net.request_per_s` as rates. \\n\\n## SOLUTION DESIGN\\n\\nThe solution was designed to:\\n- Explicitly separate gauge metrics from rate metrics, improving clarity and correctness in how metrics are collected and reported.\\n- Ensure that the correct metrics are calculated and reported in the way that aligns with the expected definitions used by the monitoring system.\\n\\nThis approach was preferred over simply fixing the existing code because it allowed for better organization and future scalability. The separation of metric types makes it easier to manage and extend.\\n\\n## IMPLEMENTATION REASONING\\n\\n1. **Version Update**:\\n ```python\\n __version__ = \"1.1.2\"\\n ```\\n - Incrementing the version indicates a significant change in functionality that should be noted by users.\\n\\n2. **Defining RATES**:\\n ```python\\n RATES = {\\n \\'Total kBytes\\': \\'apache.net.bytes_per_s\\',\\n \\'Total Accesses\\': \\'apache.net.request_per_s\\'\\n }\\n ```\\n - Clearly defining the metrics that should be treated as rates ensures that the correct calculations are applied.\\n\\n3. **Updating the Metric Collection Logic**:\\n ```python\\n if metric in self.RATES:\\n metric_count += 1\\n metric_name = self.RATES[metric]\\n self.rate(metric_name, value, tags=tags)\\n ```\\n - This change ensures that when a metric is recognized as a rate, it is processed correctly using the `self.rate()` method, allowing for proper rate calculation.\\n\\n## VERIFICATION STRATEGY\\n\\nTo verify the fix:\\n- Test the updated metrics in a local or staging environment to ensure they are correctly reported as expected.\\n- Compare the output against expected values based on known server load and traffic patterns.\\n- Conduct load tests to ensure metrics react correctly under different conditions.\\n- Check edge cases, such as very low traffic scenarios, to ensure metrics do not report erroneous values like zero rates.\\n\\n## PREVENTION & LEARNING\\n\\nTo prevent similar issues in the future:\\n- Implement unit tests for metric collection to catch incorrect mappings or calculations before deployment.\\n- Adopt a metric design pattern that clearly separates different types of metrics (e.g., gauges vs. rates) and ensure that new metrics adhere to this structure.\\n- Regularly review and refactor code related to metrics to maintain clarity and correctness as the application evolves. \\n\\nThis case highlights the importance of clear metric definitions and the need for structured code when dealing with performance monitoring, which can significantly affect user trust and system reliability.<|im_end|>\\n'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode(trainer.train_dataset[5][\"input_ids\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 140 + }, + "id": "_rD6fl8EUxnG", + "outputId": "b1da5770-3de6-48d5-c4dc-890075d0b6b0" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "' ## INITIAL PROBLEM ASSESSMENT\\n\\nThe bug report indicates that there was an issue with metric collection for Apache performance metrics, specifically `apache.net.bytes_per_s` and `apache.net.request_per_s`. This suggests that the application was not reporting accurate or expected values for these metrics, which could lead to misinterpretation of server performance by users relying on this data.\\n\\nSymptoms might include:\\n- Users not seeing expected values in monitoring dashboards.\\n- Discrepancies in data reported over time, potentially leading to false conclusions about server load and performance.\\n- Customer complaints about the monitoring system\\'s reliability.\\n\\n## HYPOTHESIS FORMATION\\n\\n1. **Incorrect Metric Mapping**: The metrics `apache.net.bytes_per_s` and `apache.net.request_per_s` might not be correctly mapped in the code, leading to values not being collected or reported as intended.\\n \\n2. **Improper Metric Calculation**: There may be a logical error in the way metrics are calculated or aggregated, such as not accounting for rates correctly.\\n\\n3. **Configuration Issues**: There could be an issue with how Apache is configured, potentially affecting what metrics are exposed and collected by the monitoring tool.\\n\\n**Ranked by likelihood**: \\n1. Incorrect Metric Mapping \\n2. Improper Metric Calculation \\n3. Configuration Issues\\n\\n## SYSTEMATIC INVESTIGATION\\n\\n1. **Review Code for Metric Mapping**:\\n - Examine the sections of code where metrics are defined and mapped to ensure that `apache.net.bytes_per_s` and `apache.net.request_per_s` are correctly included in the collection logic.\\n - Look for any recent changes in the code that might have affected the mapping.\\n\\n2. **Check Metric Calculation Logic**:\\n - Investigate how the metrics are being calculated. Look for functions or methods that aggregate or compute the values for these metrics.\\n - Confirm if the metrics are being treated as rates (per second) and check the logic for calculating rates.\\n\\n3. **Analyze Apache Configuration**:\\n - Validate the Apache configuration to ensure that the necessary modules (like `mod_status`) are enabled and that the URLs used for metric collection are correct.\\n - Check if the Apache server is correctly exposing the expected data needed for the metrics.\\n\\n## ROOT CAUSE IDENTIFICATION\\n\\nUpon reviewing the code changes in the provided patch, it becomes evident that the root cause was related to the way metrics were structured and collected. The initial code had the metrics defined in one section but did not correctly leverage them in the main collection logic. The patch introduced a dedicated section for rates (`RATES`), clearly defining how to collect `apache.net.bytes_per_s` and `apache.net.request_per_s` as rates. \\n\\n## SOLUTION DESIGN\\n\\nThe solution was designed to:\\n- Explicitly separate gauge metrics from rate metrics, improving clarity and correctness in how metrics are collected and reported.\\n- Ensure that the correct metrics are calculated and reported in the way that aligns with the expected definitions used by the monitoring system.\\n\\nThis approach was preferred over simply fixing the existing code because it allowed for better organization and future scalability. The separation of metric types makes it easier to manage and extend.\\n\\n## IMPLEMENTATION REASONING\\n\\n1. **Version Update**:\\n ```python\\n __version__ = \"1.1.2\"\\n ```\\n - Incrementing the version indicates a significant change in functionality that should be noted by users.\\n\\n2. **Defining RATES**:\\n ```python\\n RATES = {\\n \\'Total kBytes\\': \\'apache.net.bytes_per_s\\',\\n \\'Total Accesses\\': \\'apache.net.request_per_s\\'\\n }\\n ```\\n - Clearly defining the metrics that should be treated as rates ensures that the correct calculations are applied.\\n\\n3. **Updating the Metric Collection Logic**:\\n ```python\\n if metric in self.RATES:\\n metric_count += 1\\n metric_name = self.RATES[metric]\\n self.rate(metric_name, value, tags=tags)\\n ```\\n - This change ensures that when a metric is recognized as a rate, it is processed correctly using the `self.rate()` method, allowing for proper rate calculation.\\n\\n## VERIFICATION STRATEGY\\n\\nTo verify the fix:\\n- Test the updated metrics in a local or staging environment to ensure they are correctly reported as expected.\\n- Compare the output against expected values based on known server load and traffic patterns.\\n- Conduct load tests to ensure metrics react correctly under different conditions.\\n- Check edge cases, such as very low traffic scenarios, to ensure metrics do not report erroneous values like zero rates.\\n\\n## PREVENTION & LEARNING\\n\\nTo prevent similar issues in the future:\\n- Implement unit tests for metric collection to catch incorrect mappings or calculations before deployment.\\n- Adopt a metric design pattern that clearly separates different types of metrics (e.g., gauges vs. rates) and ensure that new metrics adhere to this structure.\\n- Regularly review and refactor code related to metrics to maintain clarity and correctness as the application evolves. \\n\\nThis case highlights the importance of clear metric definitions and the need for structured code when dealing with performance monitoring, which can significantly affect user trust and system reliability.<|im_end|>\\n'" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "space = tokenizer(\" \", add_special_tokens = False).input_ids[0]\n", + "tokenizer.decode([space if x == -100 else x for x in trainer.train_dataset[5][\"labels\"]])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3enWUM0jV-jV" + }, + "source": [ + "We can see the System and Instruction prompts are successfully masked!" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2ejIt2xSNKKp", + "outputId": "faf37c55-a0ad-41a3-aeed-55ee7d7070e0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPU = NVIDIA A100-SXM4-80GB. Max memory = 79.254 GB.\n", + "39.311 GB of memory reserved.\n" + ] + } + ], + "source": [ + "# @title Show current memory stats\n", + "gpu_stats = torch.cuda.get_device_properties(0)\n", + "start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n", + "print(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\n", + "print(f\"{start_gpu_memory} GB of memory reserved.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r3WJRxG-zoeS" + }, + "source": [ + "We fixed a major gradient accumulation bug in all trainers. See [blog](https://unsloth.ai/blog/gradient) for more details." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "yqxqAZ7KJ4oL", + "outputId": "f401195d-1ff9-4a9f-8a8e-ccda6bf3c0ec" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", + " \\\\ /| Num examples = 10,000 | Num Epochs = 1 | Total steps = 30\n", + "O^O/ \\_/ \\ Batch size per device = 1 | Gradient accumulation steps = 4\n", + "\\ / Data Parallel GPUs = 1 | Total batch size (1 x 4 x 1) = 4\n", + " \"-____-\" Trainable parameters = 210,534,400 of 72,916,738,048 (0.29% trained)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Will smartly offload gradients to save VRAM!\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [30/30 09:43, Epoch 0/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
11.556900
21.601600
31.493700
41.538400
51.406000
61.220800
71.155600
81.113700
91.064300
101.042700
111.052500
120.971400
131.036500
141.033200
151.030700
160.990600
170.984900
180.959900
191.013500
201.064700
210.911000
220.931300
230.924600
241.008000
250.970500
260.935600
270.917000
280.846900
290.939000
300.901200

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "trainer_stats = trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pCqnaKmlO1U9", + "outputId": "23a21d88-0604-495f-eb05-92c34ce64765" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "608.5086 seconds used for training.\n", + "10.14 minutes used for training.\n", + "Peak reserved memory = 45.48 GB.\n", + "Peak reserved memory for training = 6.169 GB.\n", + "Peak reserved memory % of max memory = 57.385 %.\n", + "Peak reserved memory for training % of max memory = 7.784 %.\n" + ] + } + ], + "source": [ + "# @title Show final memory and time stats\n", + "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n", + "used_percentage = round(used_memory / max_memory * 100, 3)\n", + "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n", + "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n", + "print(\n", + " f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n", + ")\n", + "print(f\"Peak reserved memory = {used_memory} GB.\")\n", + "print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n", + "print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n", + "print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ekOmTR1hSNcr" + }, + "source": [ + "\n", + "### Inference\n", + "Let's run the model! You can change the instruction and input - leave the output blank!\n", + "\n", + "\n", + "\n", + "We use `min_p = 0.1` and `temperature = 1.5`. Read this [Tweet](https://x.com/menhguin/status/1826132708508213629) for more information on why." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'conversation': [{'content': 'You are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.',\n", + " 'role': 'system'},\n", + " {'content': 'I\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\nElasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n\\n\\nCan you help me debug this systematically and explain your reasoning?',\n", + " 'role': 'user'},\n", + " {'content': '', 'role': 'assistant'}],\n", + " 'metadata': {'base_commit': '160cfef6e1118061fa66d333e8c2a572f5d0a815',\n", + " 'dataset': 'princeton-nlp/SWE-bench',\n", + " 'generation_timestamp': '2025-08-19T23:35:38.773997',\n", + " 'has_patch': True,\n", + " 'instance_id': 'DataDog__integrations-core-10093',\n", + " 'original_index': 0,\n", + " 'reasoning_length': 0,\n", + " 'repo': 'DataDog/integrations-core',\n", + " 'source': 'swe-bench'},\n", + " 'original_example': {'instance_id': 'DataDog__integrations-core-10093',\n", + " 'patch': \"diff --git a/elastic/datadog_checks/elastic/elastic.py b/elastic/datadog_checks/elastic/elastic.py\\n--- a/elastic/datadog_checks/elastic/elastic.py\\n+++ b/elastic/datadog_checks/elastic/elastic.py\\n@@ -139,11 +139,17 @@ def _get_es_version(self):\\n try:\\n data = self._get_data(self._config.url, send_sc=False)\\n raw_version = data['version']['number']\\n+\\n self.set_metadata('version', raw_version)\\n # pre-release versions of elasticearch are suffixed with -rcX etc..\\n # peel that off so that the map below doesn't error out\\n raw_version = raw_version.split('-')[0]\\n version = [int(p) for p in raw_version.split('.')[0:3]]\\n+ if data['version'].get('distribution', '') == 'opensearch':\\n+ # Opensearch API is backwards compatible with ES 7.10.0\\n+ # https://opensearch.org/faq\\n+ self.log.debug('OpenSearch version %s detected', version)\\n+ version = [7, 10, 0]\\n except AuthenticationError:\\n raise\\n except Exception as e:\\n\",\n", + " 'problem_statement': 'Elasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n',\n", + " 'repo': 'DataDog/integrations-core'},\n", + " 'text': '<|im_start|>system\\nYou are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.<|im_end|>\\n<|im_start|>user\\nI\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\nElasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n\\n\\nCan you help me debug this systematically and explain your reasoning?<|im_end|>\\n<|im_start|>assistant\\n<|im_end|>\\n'}" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kR3gIAX-SM2q", + "outputId": "61a9d3ff-1ecd-4af2-a3c7-a17045af4fd0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n<|im_start|>user\\nHelp me resolve the code: see the error I was getting\\n\\n#!/usr/bin/env python3\\n\"\"\"\\nStyling Inference Script\\nProvides a command-line interface to run the styling inference pipeline\\n\"\"\"\\n\\nimport sys\\nimport os\\nimport subprocess\\nimport argparse\\nfrom pathlib import Path\\n\\ndef run_inference_with_config(config_path: str, instruction: str, input_text: str = \"\", max_tokens: int = 128, stream: bool = False):\\n \"\"\"Run inference using a YAML configuration file\"\"\"\\n print(f\"Running styling inference with config: {config_path}\")\\n print(f\"Instruction: {instruction}\")\\n print(f\"Input text: {input_text}\")\\n print(f\"Max tokens: {max_tokens}\")\\n print(f\"Streaming: {stream}\")\\n\\n cmd = [\\n \"python\", \"pipelines/styling/inference.py\",\\n \"--config\", config_path,\\n \"--instruction\", instruction,\\n \"--input-text\", input_text,\\n \"--max-tokens\", str(max_tokens)\\n ]\\n\\n if stream:\\n cmd.append(\"--stream\")\\n\\n print(f\"Running: {\\' \\'.join(cmd)}\")\\n\\n try:\\n result = subprocess.run(cmd, capture_output=True, text=True, check=True)\\n print(\"✅ Inference completed successfully!\")\\n print(\"Output:\")\\n print(result.stdout)\\n return result.stdout\\n except subprocess.CalledProcessError as e:\\n print(f\"❌ Inference failed: {e}\")\\n print(\"Error output:\")\\n print(e.stderr)\\n return None\\n\\ndef run_batch_inference_example(config_path: str, input_file: str, output_file: str, instruction: str, max_tokens: int = 128):\\n \"\"\"Run batch inference example\"\"\"\\n print(f\"=== Batch Inference Example ===\")\\n print(f\"Config: {config_path}\")\\n print(f\"Input file: {input_file}\")\\n print(f\"Output file: {output_file}\")\\n print(f\"Instruction: {instruction}\")\\n print(f\"Max tokens: {max_tokens}\")\\n\\n # Check if input file exists\\n if not Path(input_file).exists():\\n print(f\"❌ Input file not found: {input_file}\")\\n return False\\n\\n # Read input texts\\n with open(input_file, \\'r\\', encoding=\\'utf-8\\') as f:\\n input_texts = [line.strip() for line in f if line.strip()]\\n\\n print(f\"Found {len(input_texts)} texts to process\")\\n\\n # Process each text\\n results = []\\n for i, text in enumerate(input_texts):\\n print(f\"\\\\nProcessing text {i+1}/{len(input_texts)}: {text[:50]}...\")\\n result = run_inference_with_config(config_path, instruction, text, max_tokens)\\n if result:\\n results.append(result)\\n else:\\n print(f\"❌ Failed to process text {i+1}\")\\n\\n # Save results\\n with open(output_file, \\'w\\', encoding=\\'utf-8\\') as f:\\n for i, (input_text, result) in enumerate(zip(input_texts, results)):\\n f.write(f\"Input {i+1}: {input_text}\\\\n\")\\n f.write(f\"Output {i+1}: {result}\\\\n\")\\n f.write(\"-\" * 50 + \"\\\\n\")\\n\\n print(f\"✅ Batch inference completed! Results saved to: {output_file}\")\\n return True\\n\\ndef show_inference_features():\\n \"\"\"Show the features of the styling inference pipeline\"\"\"\\n print(\"=== Styling Inference Pipeline Features ===\")\\n print()\\n print(\"1. **Model Support**:\")\\n print(\" - Trained LoRA models from training pipeline\")\\n print(\" - Automatic model loading from config\")\\n print(\" - Native Unsloth inference optimization\")\\n print()\\n print(\"2. **Inference Modes**:\")\\n print(\" - Single text inference with instruction\")\\n print(\" - Batch file processing\")\\n print(\" - Streaming generation\")\\n print()\\n print(\"3. **Generation Control**:\")\\n print(\" - Configurable max tokens\")\\n print(\" - Same alpaca prompt format as training\")\\n print(\" - Automatic response extraction\")\\n print()\\n print(\"4. **Usage Examples**:\")\\n print(\" - Single inference: --instruction \\'style instruction\\' --input-text \\'your text\\'\")\\n print(\" - Streaming: add --stream flag\")\\n print(\" - Batch: use batch subcommand with input/output files\")\\n\\ndef create_inference_example():\\n \"\"\"Create an inference example using the formal style configuration\"\"\"\\n print(\"=== Inference Example: Formal Style Transfer ===\")\\n print()\\n\\n # Check if we have the required files\\n config_path = \"configs/styling/formal.yaml\"\\n\\n if not Path(config_path).exists():\\n print(f\"❌ Configuration file not found: {config_path}\")\\n print(\" Please run the data processor first to create the configuration\")\\n return False\\n\\n print(\"✅ Found configuration file!\")\\n print(f\" Config: {config_path}\")\\n print()\\n\\n # Example text\\n example_text = \"Hey, what\\'s up? I\\'m gonna go grab some food later.\"\\n\\n print(f\"Example text: {example_text}\")\\n print()\\n\\n # Run inference\\n success = run_inference_with_config(\\n config_path=config_path,\\n instruction=\"Rewrite the following text in a formal style\",\\n input_text=example_text\\n )\\n\\n if success:\\n print(\"✅ Example inference completed successfully!\")\\n return True\\n else:\\n print(\"❌ Example inference failed!\")\\n return False\\n\\ndef main():\\n \"\"\"Main inference function\"\"\"\\n parser = argparse.ArgumentParser(description=\"Styling Inference Pipeline\")\\n subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\\n\\n # Inference command\\n infer_parser = subparsers.add_parser(\"infer\", help=\"Run single inference\")\\n infer_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\\n infer_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\\n infer_parser.add_argument(\"--input-text\", type=str, default=\"\", help=\"Input text to style\")\\n infer_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\\n infer_parser.add_argument(\"--stream\", action=\"store_true\", help=\"Enable streaming generation\")\\n\\n # Batch inference command\\n batch_parser = subparsers.add_parser(\"batch\", help=\"Run batch inference\")\\n batch_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\\n batch_parser.add_argument(\"--input-file\", type=str, required=True, help=\"Input file with texts to style\")\\n batch_parser.add_argument(\"--output-file\", type=str, required=True, help=\"Output file for styled texts\")\\n batch_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\\n batch_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\\n\\n # Features command\\n subparsers.add_parser(\"features\", help=\"Show available features\")\\n\\n # Example command\\n subparsers.add_parser(\"example\", help=\"Run example inference\")\\n\\n args = parser.parse_args()\\n\\n if args.command == \"infer\":\\n run_inference_with_config(\\n args.config, \\n args.instruction, \\n args.input_text, \\n args.max_tokens, \\n args.stream\\n )\\n elif args.command == \"batch\":\\n run_batch_inference_example(args.config, args.input_file, args.output_file, args.instruction, args.max_tokens)\\n elif args.command == \"features\":\\n show_inference_features()\\n elif args.command == \"example\":\\n create_inference_example()\\n else:\\n parser.print_help()\\n\\nif __name__ == \"__main__\":\\n main()\\n\\nERROR: \\n(base) macbook@DynamoAi fine-tune-task % python scripts/styling/inference.py\\nusage: inference.py [-h] {infer,batch,features,example} ...\\n\\nStyling Inference Pipeline\\n\\npositional arguments:\\n {infer,batch,features,example}\\n Available commands\\n infer Run single inference\\n batch Run batch inference\\n features Show available features\\n example Run example inference\\n\\noptions:\\n -h, --help show this help message and exit\\n(base) macbook@DynamoAi fine-tune-task %<|im_end|>\\n<|im_start|>assistant\\nThe error you encountered is due to the absence of a command argument when running the script `inference.py`. Your script is designed to be used in conjunction with various subcommands such as `infer`, `batch`, `features`, or `example`. \\n\\nHere\\'s how these subcommands work:\\n\\n1. `infer`:']\n" + ] + } + ], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "\n", + "# Apply chat template to tokenizer\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template=\"qwen-2.5\",\n", + ")\n", + "\n", + "# Enable faster inference\n", + "FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n", + "\n", + "# Define the messages with properly formatted content\n", + "messages = [\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"\"\"Help me resolve the code: see the error I was getting\n", + "\n", + "#!/usr/bin/env python3\n", + "\\\"\\\"\\\"\n", + "Styling Inference Script\n", + "Provides a command-line interface to run the styling inference pipeline\n", + "\\\"\\\"\\\"\n", + "\n", + "import sys\n", + "import os\n", + "import subprocess\n", + "import argparse\n", + "from pathlib import Path\n", + "\n", + "def run_inference_with_config(config_path: str, instruction: str, input_text: str = \"\", max_tokens: int = 128, stream: bool = False):\n", + " \\\"\\\"\\\"Run inference using a YAML configuration file\\\"\\\"\\\"\n", + " print(f\"Running styling inference with config: {config_path}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Input text: {input_text}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " print(f\"Streaming: {stream}\")\n", + " \n", + " cmd = [\n", + " \"python\", \"pipelines/styling/inference.py\",\n", + " \"--config\", config_path,\n", + " \"--instruction\", instruction,\n", + " \"--input-text\", input_text,\n", + " \"--max-tokens\", str(max_tokens)\n", + " ]\n", + " \n", + " if stream:\n", + " cmd.append(\"--stream\")\n", + " \n", + " print(f\"Running: {' '.join(cmd)}\")\n", + " \n", + " try:\n", + " result = subprocess.run(cmd, capture_output=True, text=True, check=True)\n", + " print(\"✅ Inference completed successfully!\")\n", + " print(\"Output:\")\n", + " print(result.stdout)\n", + " return result.stdout\n", + " except subprocess.CalledProcessError as e:\n", + " print(f\"❌ Inference failed: {e}\")\n", + " print(\"Error output:\")\n", + " print(e.stderr)\n", + " return None\n", + "\n", + "def run_batch_inference_example(config_path: str, input_file: str, output_file: str, instruction: str, max_tokens: int = 128):\n", + " \\\"\\\"\\\"Run batch inference example\\\"\\\"\\\"\n", + " print(f\"=== Batch Inference Example ===\")\n", + " print(f\"Config: {config_path}\")\n", + " print(f\"Input file: {input_file}\")\n", + " print(f\"Output file: {output_file}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " \n", + " # Check if input file exists\n", + " if not Path(input_file).exists():\n", + " print(f\"❌ Input file not found: {input_file}\")\n", + " return False\n", + " \n", + " # Read input texts\n", + " with open(input_file, 'r', encoding='utf-8') as f:\n", + " input_texts = [line.strip() for line in f if line.strip()]\n", + " \n", + " print(f\"Found {len(input_texts)} texts to process\")\n", + " \n", + " # Process each text\n", + " results = []\n", + " for i, text in enumerate(input_texts):\n", + " print(f\"\\\\nProcessing text {i+1}/{len(input_texts)}: {text[:50]}...\")\n", + " result = run_inference_with_config(config_path, instruction, text, max_tokens)\n", + " if result:\n", + " results.append(result)\n", + " else:\n", + " print(f\"❌ Failed to process text {i+1}\")\n", + " \n", + " # Save results\n", + " with open(output_file, 'w', encoding='utf-8') as f:\n", + " for i, (input_text, result) in enumerate(zip(input_texts, results)):\n", + " f.write(f\"Input {i+1}: {input_text}\\\\n\")\n", + " f.write(f\"Output {i+1}: {result}\\\\n\")\n", + " f.write(\"-\" * 50 + \"\\\\n\")\n", + " \n", + " print(f\"✅ Batch inference completed! Results saved to: {output_file}\")\n", + " return True\n", + "\n", + "def show_inference_features():\n", + " \\\"\\\"\\\"Show the features of the styling inference pipeline\\\"\\\"\\\"\n", + " print(\"=== Styling Inference Pipeline Features ===\")\n", + " print()\n", + " print(\"1. **Model Support**:\")\n", + " print(\" - Trained LoRA models from training pipeline\")\n", + " print(\" - Automatic model loading from config\")\n", + " print(\" - Native Unsloth inference optimization\")\n", + " print()\n", + " print(\"2. **Inference Modes**:\")\n", + " print(\" - Single text inference with instruction\")\n", + " print(\" - Batch file processing\")\n", + " print(\" - Streaming generation\")\n", + " print()\n", + " print(\"3. **Generation Control**:\")\n", + " print(\" - Configurable max tokens\")\n", + " print(\" - Same alpaca prompt format as training\")\n", + " print(\" - Automatic response extraction\")\n", + " print()\n", + " print(\"4. **Usage Examples**:\")\n", + " print(\" - Single inference: --instruction 'style instruction' --input-text 'your text'\")\n", + " print(\" - Streaming: add --stream flag\")\n", + " print(\" - Batch: use batch subcommand with input/output files\")\n", + "\n", + "def create_inference_example():\n", + " \\\"\\\"\\\"Create an inference example using the formal style configuration\\\"\\\"\\\"\n", + " print(\"=== Inference Example: Formal Style Transfer ===\")\n", + " print()\n", + " \n", + " # Check if we have the required files\n", + " config_path = \"configs/styling/formal.yaml\"\n", + " \n", + " if not Path(config_path).exists():\n", + " print(f\"❌ Configuration file not found: {config_path}\")\n", + " print(\" Please run the data processor first to create the configuration\")\n", + " return False\n", + " \n", + " print(\"✅ Found configuration file!\")\n", + " print(f\" Config: {config_path}\")\n", + " print()\n", + " \n", + " # Example text\n", + " example_text = \"Hey, what's up? I'm gonna go grab some food later.\"\n", + " \n", + " print(f\"Example text: {example_text}\")\n", + " print()\n", + " \n", + " # Run inference\n", + " success = run_inference_with_config(\n", + " config_path=config_path,\n", + " instruction=\"Rewrite the following text in a formal style\",\n", + " input_text=example_text\n", + " )\n", + " \n", + " if success:\n", + " print(\"✅ Example inference completed successfully!\")\n", + " return True\n", + " else:\n", + " print(\"❌ Example inference failed!\")\n", + " return False\n", + "\n", + "def main():\n", + " \\\"\\\"\\\"Main inference function\\\"\\\"\\\"\n", + " parser = argparse.ArgumentParser(description=\"Styling Inference Pipeline\")\n", + " subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\n", + " \n", + " # Inference command\n", + " infer_parser = subparsers.add_parser(\"infer\", help=\"Run single inference\")\n", + " infer_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " infer_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " infer_parser.add_argument(\"--input-text\", type=str, default=\"\", help=\"Input text to style\")\n", + " infer_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " infer_parser.add_argument(\"--stream\", action=\"store_true\", help=\"Enable streaming generation\")\n", + " \n", + " # Batch inference command\n", + " batch_parser = subparsers.add_parser(\"batch\", help=\"Run batch inference\")\n", + " batch_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " batch_parser.add_argument(\"--input-file\", type=str, required=True, help=\"Input file with texts to style\")\n", + " batch_parser.add_argument(\"--output-file\", type=str, required=True, help=\"Output file for styled texts\")\n", + " batch_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " batch_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " \n", + " # Features command\n", + " subparsers.add_parser(\"features\", help=\"Show available features\")\n", + " \n", + " # Example command\n", + " subparsers.add_parser(\"example\", help=\"Run example inference\")\n", + " \n", + " args = parser.parse_args()\n", + " \n", + " if args.command == \"infer\":\n", + " run_inference_with_config(\n", + " args.config, \n", + " args.instruction, \n", + " args.input_text, \n", + " args.max_tokens, \n", + " args.stream\n", + " )\n", + " elif args.command == \"batch\":\n", + " run_batch_inference_example(args.config, args.input_file, args.output_file, args.instruction, args.max_tokens)\n", + " elif args.command == \"features\":\n", + " show_inference_features()\n", + " elif args.command == \"example\":\n", + " create_inference_example()\n", + " else:\n", + " parser.print_help()\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n", + "\n", + "ERROR: \n", + "(base) macbook@DynamoAi fine-tune-task % python scripts/styling/inference.py\n", + "usage: inference.py [-h] {infer,batch,features,example} ...\n", + "\n", + "Styling Inference Pipeline\n", + "\n", + "positional arguments:\n", + " {infer,batch,features,example}\n", + " Available commands\n", + " infer Run single inference\n", + " batch Run batch inference\n", + " features Show available features\n", + " example Run example inference\n", + "\n", + "options:\n", + " -h, --help show this help message and exit\n", + "(base) macbook@DynamoAi fine-tune-task %\"\"\"\n", + " },\n", + "]\n", + "\n", + "# Apply chat template and tokenize\n", + "inputs = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=True,\n", + " add_generation_prompt=True, # Must add for generation\n", + " return_tensors=\"pt\",\n", + ").to(\"cuda\")\n", + "\n", + "# Generate response\n", + "outputs = model.generate(\n", + " input_ids=inputs, \n", + " max_new_tokens=64, \n", + " use_cache=True,\n", + " temperature=1.5, \n", + " min_p=0.1\n", + ")\n", + "\n", + "# Decode the outputs\n", + "result = tokenizer.batch_decode(outputs)\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CrSvZObor0lY" + }, + "source": [ + " You can also use a `TextStreamer` for continuous inference - so you can see the generation token by token, instead of waiting the whole time!" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "e2pEuRb1r2Vg", + "outputId": "fe6590c1-dbd8-475b-c99b-17f88b302f9d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The error you're encountering is due to the fact that the script expects specific subcommands to be provided when running it. The error message indicates that no positional argument (subcommand) was given, which is required based on how the `argparse` module is set up in your script.\n", + "\n", + "### Explanation of the Error:\n", + "\n", + "When you run the script without any subcommands, the `args.command` will be `None`, leading to the `else` block where the help message is printed. This is why you see the usage information instead of the actual execution of the script.\n", + "\n", + "### Solution:\n", + "\n", + "To resolve this issue, you need to provide one of the expected subcommands when running the script. Here are the valid subcommands:\n", + "\n", + "1. **infer**: For running a single inference.\n", + "2. **batch**: For running batch inference.\n", + "3. **features**: To show the available features.\n", + "4. **example**: To run an example inference.\n", + "\n", + "### How to Run the Script Correctly:\n", + "\n", + "#### 1. Running Single Inference:\n", + "```bash\n", + "python scripts/styling/inference.py infer --config path/to/config.yaml --instruction \"Your instruction\" --input-text \"Your input text\"\n", + "```\n", + "\n", + "#### 2. Running Batch Inference:\n", + "```bash\n", + "python scripts/styling/inference.py batch --config path/to/config.yaml --input-file path/to/input.txt --output-file path/to/output.txt --instruction \"Your instruction\"\n", + "```\n", + "\n", + "#### 3. Showing Features:\n", + "```bash\n", + "python scripts/styling/inference.py features\n", + "```\n", + "\n", + "#### 4. Running Example Inference:\n", + "```bash\n", + "python scripts/styling/inference.py example\n", + "```\n", + "\n", + "### Additional Notes:\n", + "\n", + "- Ensure that the paths provided for `--config`, `--input-file`, and `--output-file` are correct and accessible.\n", + "- If you encounter any issues with file paths or configurations, check the existence of the files and the correctness of the YAML configuration.\n", + "\n", + "By providing the appropriate subcommand, you should be able to execute the script without encountering the error.<|im_end|>\n" + ] + } + ], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "\n", + "# Apply chat template to tokenizer\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template=\"qwen-2.5\",\n", + ")\n", + "\n", + "# Enable faster inference\n", + "FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n", + "\n", + "# Define the messages with properly formatted content\n", + "messages = [\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"\"\"Help me resolve the code: see the error I was getting\n", + "\n", + "#!/usr/bin/env python3\n", + "\\\"\\\"\\\"\n", + "Styling Inference Script\n", + "Provides a command-line interface to run the styling inference pipeline\n", + "\\\"\\\"\\\"\n", + "\n", + "import sys\n", + "import os\n", + "import subprocess\n", + "import argparse\n", + "from pathlib import Path\n", + "\n", + "def run_inference_with_config(config_path: str, instruction: str, input_text: str = \"\", max_tokens: int = 128, stream: bool = False):\n", + " \\\"\\\"\\\"Run inference using a YAML configuration file\\\"\\\"\\\"\n", + " print(f\"Running styling inference with config: {config_path}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Input text: {input_text}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " print(f\"Streaming: {stream}\")\n", + " \n", + " cmd = [\n", + " \"python\", \"pipelines/styling/inference.py\",\n", + " \"--config\", config_path,\n", + " \"--instruction\", instruction,\n", + " \"--input-text\", input_text,\n", + " \"--max-tokens\", str(max_tokens)\n", + " ]\n", + " \n", + " if stream:\n", + " cmd.append(\"--stream\")\n", + " \n", + " print(f\"Running: {' '.join(cmd)}\")\n", + " \n", + " try:\n", + " result = subprocess.run(cmd, capture_output=True, text=True, check=True)\n", + " print(\"✅ Inference completed successfully!\")\n", + " print(\"Output:\")\n", + " print(result.stdout)\n", + " return result.stdout\n", + " except subprocess.CalledProcessError as e:\n", + " print(f\"❌ Inference failed: {e}\")\n", + " print(\"Error output:\")\n", + " print(e.stderr)\n", + " return None\n", + "\n", + "def run_batch_inference_example(config_path: str, input_file: str, output_file: str, instruction: str, max_tokens: int = 128):\n", + " \\\"\\\"\\\"Run batch inference example\\\"\\\"\\\"\n", + " print(f\"=== Batch Inference Example ===\")\n", + " print(f\"Config: {config_path}\")\n", + " print(f\"Input file: {input_file}\")\n", + " print(f\"Output file: {output_file}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " \n", + " # Check if input file exists\n", + " if not Path(input_file).exists():\n", + " print(f\"❌ Input file not found: {input_file}\")\n", + " return False\n", + " \n", + " # Read input texts\n", + " with open(input_file, 'r', encoding='utf-8') as f:\n", + " input_texts = [line.strip() for line in f if line.strip()]\n", + " \n", + " print(f\"Found {len(input_texts)} texts to process\")\n", + " \n", + " # Process each text\n", + " results = []\n", + " for i, text in enumerate(input_texts):\n", + " print(f\"\\\\nProcessing text {i+1}/{len(input_texts)}: {text[:50]}...\")\n", + " result = run_inference_with_config(config_path, instruction, text, max_tokens)\n", + " if result:\n", + " results.append(result)\n", + " else:\n", + " print(f\"❌ Failed to process text {i+1}\")\n", + " \n", + " # Save results\n", + " with open(output_file, 'w', encoding='utf-8') as f:\n", + " for i, (input_text, result) in enumerate(zip(input_texts, results)):\n", + " f.write(f\"Input {i+1}: {input_text}\\\\n\")\n", + " f.write(f\"Output {i+1}: {result}\\\\n\")\n", + " f.write(\"-\" * 50 + \"\\\\n\")\n", + " \n", + " print(f\"✅ Batch inference completed! Results saved to: {output_file}\")\n", + " return True\n", + "\n", + "def show_inference_features():\n", + " \\\"\\\"\\\"Show the features of the styling inference pipeline\\\"\\\"\\\"\n", + " print(\"=== Styling Inference Pipeline Features ===\")\n", + " print()\n", + " print(\"1. **Model Support**:\")\n", + " print(\" - Trained LoRA models from training pipeline\")\n", + " print(\" - Automatic model loading from config\")\n", + " print(\" - Native Unsloth inference optimization\")\n", + " print()\n", + " print(\"2. **Inference Modes**:\")\n", + " print(\" - Single text inference with instruction\")\n", + " print(\" - Batch file processing\")\n", + " print(\" - Streaming generation\")\n", + " print()\n", + " print(\"3. **Generation Control**:\")\n", + " print(\" - Configurable max tokens\")\n", + " print(\" - Same alpaca prompt format as training\")\n", + " print(\" - Automatic response extraction\")\n", + " print()\n", + " print(\"4. **Usage Examples**:\")\n", + " print(\" - Single inference: --instruction 'style instruction' --input-text 'your text'\")\n", + " print(\" - Streaming: add --stream flag\")\n", + " print(\" - Batch: use batch subcommand with input/output files\")\n", + "\n", + "def create_inference_example():\n", + " \\\"\\\"\\\"Create an inference example using the formal style configuration\\\"\\\"\\\"\n", + " print(\"=== Inference Example: Formal Style Transfer ===\")\n", + " print()\n", + " \n", + " # Check if we have the required files\n", + " config_path = \"configs/styling/formal.yaml\"\n", + " \n", + " if not Path(config_path).exists():\n", + " print(f\"❌ Configuration file not found: {config_path}\")\n", + " print(\" Please run the data processor first to create the configuration\")\n", + " return False\n", + " \n", + " print(\"✅ Found configuration file!\")\n", + " print(f\" Config: {config_path}\")\n", + " print()\n", + " \n", + " # Example text\n", + " example_text = \"Hey, what's up? I'm gonna go grab some food later.\"\n", + " \n", + " print(f\"Example text: {example_text}\")\n", + " print()\n", + " \n", + " # Run inference\n", + " success = run_inference_with_config(\n", + " config_path=config_path,\n", + " instruction=\"Rewrite the following text in a formal style\",\n", + " input_text=example_text\n", + " )\n", + " \n", + " if success:\n", + " print(\"✅ Example inference completed successfully!\")\n", + " return True\n", + " else:\n", + " print(\"❌ Example inference failed!\")\n", + " return False\n", + "\n", + "def main():\n", + " \\\"\\\"\\\"Main inference function\\\"\\\"\\\"\n", + " parser = argparse.ArgumentParser(description=\"Styling Inference Pipeline\")\n", + " subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\n", + " \n", + " # Inference command\n", + " infer_parser = subparsers.add_parser(\"infer\", help=\"Run single inference\")\n", + " infer_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " infer_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " infer_parser.add_argument(\"--input-text\", type=str, default=\"\", help=\"Input text to style\")\n", + " infer_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " infer_parser.add_argument(\"--stream\", action=\"store_true\", help=\"Enable streaming generation\")\n", + " \n", + " # Batch inference command\n", + " batch_parser = subparsers.add_parser(\"batch\", help=\"Run batch inference\")\n", + " batch_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " batch_parser.add_argument(\"--input-file\", type=str, required=True, help=\"Input file with texts to style\")\n", + " batch_parser.add_argument(\"--output-file\", type=str, required=True, help=\"Output file for styled texts\")\n", + " batch_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " batch_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " \n", + " # Features command\n", + " subparsers.add_parser(\"features\", help=\"Show available features\")\n", + " \n", + " # Example command\n", + " subparsers.add_parser(\"example\", help=\"Run example inference\")\n", + " \n", + " args = parser.parse_args()\n", + " \n", + " if args.command == \"infer\":\n", + " run_inference_with_config(\n", + " args.config, \n", + " args.instruction, \n", + " args.input_text, \n", + " args.max_tokens, \n", + " args.stream\n", + " )\n", + " elif args.command == \"batch\":\n", + " run_batch_inference_example(args.config, args.input_file, args.output_file, args.instruction, args.max_tokens)\n", + " elif args.command == \"features\":\n", + " show_inference_features()\n", + " elif args.command == \"example\":\n", + " create_inference_example()\n", + " else:\n", + " parser.print_help()\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n", + "\n", + "ERROR: \n", + "(base) macbook@DynamoAi fine-tune-task % python scripts/styling/inference.py\n", + "usage: inference.py [-h] {infer,batch,features,example} ...\n", + "\n", + "Styling Inference Pipeline\n", + "\n", + "positional arguments:\n", + " {infer,batch,features,example}\n", + " Available commands\n", + " infer Run single inference\n", + " batch Run batch inference\n", + " features Show available features\n", + " example Run example inference\n", + "\n", + "options:\n", + " -h, --help show this help message and exit\n", + "(base) macbook@DynamoAi fine-tune-task %\"\"\"\n", + " },\n", + "]\n", + "\n", + "# Apply chat template and tokenize\n", + "inputs = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=True,\n", + " add_generation_prompt=True, # Must add for generation\n", + " return_tensors=\"pt\",\n", + ").to(\"cuda\")\n", + "\n", + "# Decode the outputs\n", + "result = tokenizer.batch_decode(outputs)\n", + "\n", + "from transformers import TextStreamer\n", + "text_streamer = TextStreamer(tokenizer, skip_prompt = True)\n", + "_ = model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 1024,\n", + " use_cache = True, temperature = 0.2, min_p = 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uMuVrWbjAzhc" + }, + "source": [ + "\n", + "### Saving, loading finetuned models\n", + "To save the final model as LoRA adapters, either use Huggingface's `push_to_hub` for an online save or `save_pretrained` for a local save.\n", + "\n", + "**[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down!" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "upcOlWe7A1vc", + "outputId": "04d11b2b-7efd-4598-a60a-1873cfd3c431" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e60608a0567042ae9eef2a56b1760f96", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "README.md: 0%| | 0.00/601 [00:00 \u001b[39m\u001b[32m23\u001b[39m _ = \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstreamer\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtext_streamer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_new_tokens\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m128\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 24\u001b[39m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m1.5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmin_p\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m0.1\u001b[39;49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/peft/peft_model.py:1973\u001b[39m, in \u001b[36mPeftModelForCausalLM.generate\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1971\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m._enable_peft_forward_hooks(*args, **kwargs):\n\u001b[32m 1972\u001b[39m kwargs = {k: v \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m kwargs.items() \u001b[38;5;28;01mif\u001b[39;00m k \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.special_peft_forward_args}\n\u001b[32m-> \u001b[39m\u001b[32m1973\u001b[39m outputs = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mbase_model\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1974\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1975\u001b[39m outputs = \u001b[38;5;28mself\u001b[39m.base_model.generate(**kwargs)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:1794\u001b[39m, in \u001b[36munsloth_fast_generate\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1792\u001b[39m \u001b[38;5;66;03m# Mixed precision autocast\u001b[39;00m\n\u001b[32m 1793\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m torch.inference_mode(), torch.autocast(device_type = DEVICE_TYPE, dtype = dtype):\n\u001b[32m-> \u001b[39m\u001b[32m1794\u001b[39m output = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_old_generate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1795\u001b[39m \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[32m 1797\u001b[39m \u001b[38;5;66;03m# Return accelerate back\u001b[39;00m\n\u001b[32m 1798\u001b[39m \u001b[38;5;66;03m# if accelerate_new_send_to_device is not None:\u001b[39;00m\n\u001b[32m 1799\u001b[39m \u001b[38;5;66;03m# accelerate.utils.operations.send_to_device = accelerate_old_send_to_device\u001b[39;00m\n\u001b[32m 1800\u001b[39m \u001b[38;5;66;03m# pass\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/torch/utils/_contextlib.py:120\u001b[39m, in \u001b[36mcontext_decorator..decorate_context\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 117\u001b[39m \u001b[38;5;129m@functools\u001b[39m.wraps(func)\n\u001b[32m 118\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mdecorate_context\u001b[39m(*args, **kwargs):\n\u001b[32m 119\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[32m--> \u001b[39m\u001b[32m120\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/transformers/generation/utils.py:2617\u001b[39m, in \u001b[36mGenerationMixin.generate\u001b[39m\u001b[34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, use_model_defaults, custom_generate, **kwargs)\u001b[39m\n\u001b[32m 2609\u001b[39m input_ids, model_kwargs = \u001b[38;5;28mself\u001b[39m._expand_inputs_for_generation(\n\u001b[32m 2610\u001b[39m input_ids=input_ids,\n\u001b[32m 2611\u001b[39m expand_size=generation_config.num_return_sequences,\n\u001b[32m 2612\u001b[39m is_encoder_decoder=\u001b[38;5;28mself\u001b[39m.config.is_encoder_decoder,\n\u001b[32m 2613\u001b[39m **model_kwargs,\n\u001b[32m 2614\u001b[39m )\n\u001b[32m 2616\u001b[39m \u001b[38;5;66;03m# 12. run sample (it degenerates to greedy search when `generation_config.do_sample=False`)\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m2617\u001b[39m result = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sample\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 2618\u001b[39m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2619\u001b[39m \u001b[43m \u001b[49m\u001b[43mlogits_processor\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprepared_logits_processor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2620\u001b[39m \u001b[43m \u001b[49m\u001b[43mstopping_criteria\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprepared_stopping_criteria\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2621\u001b[39m \u001b[43m \u001b[49m\u001b[43mgeneration_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgeneration_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2622\u001b[39m \u001b[43m \u001b[49m\u001b[43msynced_gpus\u001b[49m\u001b[43m=\u001b[49m\u001b[43msynced_gpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2623\u001b[39m \u001b[43m \u001b[49m\u001b[43mstreamer\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstreamer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2624\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2625\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2627\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m generation_mode \u001b[38;5;129;01min\u001b[39;00m (GenerationMode.BEAM_SAMPLE, GenerationMode.BEAM_SEARCH):\n\u001b[32m 2628\u001b[39m \u001b[38;5;66;03m# 11. interleave input_ids with `num_beams` additional sequences per batch\u001b[39;00m\n\u001b[32m 2629\u001b[39m input_ids, model_kwargs = \u001b[38;5;28mself\u001b[39m._expand_inputs_for_generation(\n\u001b[32m 2630\u001b[39m input_ids=input_ids,\n\u001b[32m 2631\u001b[39m expand_size=generation_config.num_beams,\n\u001b[32m 2632\u001b[39m is_encoder_decoder=\u001b[38;5;28mself\u001b[39m.config.is_encoder_decoder,\n\u001b[32m 2633\u001b[39m **model_kwargs,\n\u001b[32m 2634\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/transformers/generation/utils.py:3601\u001b[39m, in \u001b[36mGenerationMixin._sample\u001b[39m\u001b[34m(self, input_ids, logits_processor, stopping_criteria, generation_config, synced_gpus, streamer, **model_kwargs)\u001b[39m\n\u001b[32m 3599\u001b[39m is_prefill = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m 3600\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m3601\u001b[39m outputs = \u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m 3603\u001b[39m \u001b[38;5;66;03m# synced_gpus: don't waste resources running the code we don't need; kwargs must be updated before skipping\u001b[39;00m\n\u001b[32m 3604\u001b[39m model_kwargs = \u001b[38;5;28mself\u001b[39m._update_model_kwargs_for_generation(\n\u001b[32m 3605\u001b[39m outputs,\n\u001b[32m 3606\u001b[39m model_kwargs,\n\u001b[32m 3607\u001b[39m is_encoder_decoder=\u001b[38;5;28mself\u001b[39m.config.is_encoder_decoder,\n\u001b[32m 3608\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py:1773\u001b[39m, in \u001b[36mModule._wrapped_call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1771\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._compiled_call_impl(*args, **kwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 1772\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1773\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py:1784\u001b[39m, in \u001b[36mModule._call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1779\u001b[39m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[32m 1780\u001b[39m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[32m 1781\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_pre_hooks\n\u001b[32m 1782\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[32m 1783\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[32m-> \u001b[39m\u001b[32m1784\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1786\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1787\u001b[39m called_always_called_hooks = \u001b[38;5;28mset\u001b[39m()\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:1166\u001b[39m, in \u001b[36mCausalLM_fast_forward.._CausalLM_fast_forward\u001b[39m\u001b[34m(self, input_ids, causal_mask, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict, num_logits_to_keep, logits_to_keep, *args, **kwargs)\u001b[39m\n\u001b[32m 1148\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_CausalLM_fast_forward\u001b[39m(\n\u001b[32m 1149\u001b[39m \u001b[38;5;28mself\u001b[39m,\n\u001b[32m 1150\u001b[39m input_ids: torch.LongTensor = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m (...)\u001b[39m\u001b[32m 1163\u001b[39m *args, **kwargs,\n\u001b[32m 1164\u001b[39m ) -> Union[Tuple, CausalLMOutputWithPast]:\n\u001b[32m 1165\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m past_key_values \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1166\u001b[39m outputs = \u001b[43mfast_forward_inference\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1167\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 1168\u001b[39m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1169\u001b[39m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1170\u001b[39m \u001b[43m \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1171\u001b[39m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1172\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1173\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1174\u001b[39m causal_mask = xformers.attn_bias.LowerTriangularMask() \u001b[38;5;28;01mif\u001b[39;00m HAS_XFORMERS \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:1117\u001b[39m, in \u001b[36m_LlamaModel_fast_forward_inference..LlamaModel_fast_forward_inference_custom\u001b[39m\u001b[34m(self, input_ids, past_key_values, position_ids, attention_mask)\u001b[39m\n\u001b[32m 1109\u001b[39m residual.copy_(X) \u001b[38;5;66;03m# residual = X\u001b[39;00m\n\u001b[32m 1110\u001b[39m X = fast_rms_layernorm_inference(\n\u001b[32m 1111\u001b[39m decoder_layer.post_attention_layernorm,\n\u001b[32m 1112\u001b[39m X,\n\u001b[32m (...)\u001b[39m\u001b[32m 1115\u001b[39m variance = variance,\n\u001b[32m 1116\u001b[39m )\n\u001b[32m-> \u001b[39m\u001b[32m1117\u001b[39m X = \u001b[43mmlp_fast_forward_inference\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1118\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecoder_layer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmlp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1119\u001b[39m \u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1120\u001b[39m \u001b[43m \u001b[49m\u001b[43mtemp_gate\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_gates\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_index\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1121\u001b[39m \u001b[43m \u001b[49m\u001b[43mtemp_up\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_ups\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_index\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1122\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1123\u001b[39m X += residual\n\u001b[32m 1125\u001b[39m next_decoder_cache.append(present_key_value)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:411\u001b[39m, in \u001b[36mfast_swiglu_inference\u001b[39m\u001b[34m(self, X, temp_gate, temp_up, gate_multiplier, down_multiplier)\u001b[39m\n\u001b[32m 408\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m gate_multiplier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 409\u001b[39m gate *= gate_multiplier\n\u001b[32m--> \u001b[39m\u001b[32m411\u001b[39m up = \u001b[43mfast_linear_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m \u001b[49m\u001b[43mup_proj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_up\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 413\u001b[39m gate = torch_nn_functional_silu(gate, inplace = \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 414\u001b[39m gate *= up\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/kernels/utils.py:680\u001b[39m, in \u001b[36mfast_linear_forward\u001b[39m\u001b[34m(proj, X, temp_lora, out)\u001b[39m\n\u001b[32m 678\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m bsz == \u001b[32m1\u001b[39m:\n\u001b[32m 679\u001b[39m out = out.view(out_dim)\n\u001b[32m--> \u001b[39m\u001b[32m680\u001b[39m temp_lora = \u001b[43mtorch_mv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlora_A\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_fast_lora\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m.\u001b[49m\u001b[43mravel\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_lora\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 681\u001b[39m out.addmv_(lora_B._fast_lora, temp_lora, alpha = lora_S)\n\u001b[32m 682\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "\u001b[31mKeyboardInterrupt\u001b[39m: " + ] + } + ], + "source": [ + "if False:\n", + " from unsloth import FastLanguageModel\n", + " model, tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name = \"qwen_2_5_72b_debug_research\", # YOUR MODEL YOU USED FOR TRAINING\n", + " max_seq_length = max_seq_length,\n", + " dtype = dtype,\n", + " load_in_4bit = load_in_4bit,\n", + " )\n", + " FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n", + "\n", + "messages = [\n", + " {\"role\": \"user\", \"content\": \"Describe a tall tower in the capital of France.\"},\n", + "]\n", + "inputs = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize = True,\n", + " add_generation_prompt = True, # Must add for generation\n", + " return_tensors = \"pt\",\n", + ").to(\"cuda\")\n", + "\n", + "from transformers import TextStreamer\n", + "text_streamer = TextStreamer(tokenizer, skip_prompt = True)\n", + "_ = model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 128,\n", + " use_cache = True, temperature = 1.5, min_p = 0.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QQMjaNrjsU5_" + }, + "source": [ + "You can also use Hugging Face's `AutoModelForPeftCausalLM`. Only use this if you do not have `unsloth` installed. It can be hopelessly slow, since `4bit` model downloading is not supported, and Unsloth's **inference is 2x faster**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yFfaXG0WsQuE" + }, + "outputs": [], + "source": [ + "if False:\n", + " # I highly do NOT suggest - use Unsloth if possible\n", + " from peft import AutoPeftModelForCausalLM\n", + " from transformers import AutoTokenizer\n", + "\n", + " model = AutoPeftModelForCausalLM.from_pretrained(\n", + " \"lora_model\", # YOUR MODEL YOU USED FOR TRAINING\n", + " load_in_4bit=load_in_4bit,\n", + " )\n", + " tokenizer = AutoTokenizer.from_pretrained(\"lora_model\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f422JgM9sdVT" + }, + "source": [ + "### Saving to float16 for VLLM\n", + "\n", + "We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "iHjt_SMYsd3P" + }, + "outputs": [], + "source": [ + "# Merge to 16bit\n", + "if False: model.save_pretrained_merged(\"model\", tokenizer, save_method = \"merged_16bit\",)\n", + "if False: model.push_to_hub_merged(\"hf/model\", tokenizer, save_method = \"merged_16bit\", token = \"\")\n", + "\n", + "# Merge to 4bit\n", + "if False: model.save_pretrained_merged(\"model\", tokenizer, save_method = \"merged_4bit\",)\n", + "if False: model.push_to_hub_merged(\"hf/model\", tokenizer, save_method = \"merged_4bit\", token = \"\")\n", + "\n", + "# Just LoRA adapters\n", + "if False:\n", + " model.save_pretrained(\"model\")\n", + " tokenizer.save_pretrained(\"model\")\n", + "if False:\n", + " model.push_to_hub(\"hf/model\", token = \"\")\n", + " tokenizer.push_to_hub(\"hf/model\", token = \"\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TCv4vXHd61i7" + }, + "source": [ + "### GGUF / llama.cpp Conversion\n", + "To save to `GGUF` / `llama.cpp`, we support it natively now! We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF.\n", + "\n", + "Some supported quant methods (full list on our [Wiki page](https://github.com/unslothai/unsloth/wiki#gguf-quantization-options)):\n", + "* `q8_0` - Fast conversion. High resource use, but generally acceptable.\n", + "* `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K.\n", + "* `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K.\n", + "\n", + "[**NEW**] To finetune and auto export to Ollama, try our [Ollama notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FqfebeAdT073" + }, + "outputs": [], + "source": [ + "# Save to 8bit Q8_0\n", + "if False: model.save_pretrained_gguf(\"model\", tokenizer,)\n", + "# Remember to go to https://huggingface.co/settings/tokens for a token!\n", + "# And change hf to your username!\n", + "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, token = \"\")\n", + "\n", + "# Save to 16bit GGUF\n", + "if False: model.save_pretrained_gguf(\"model\", tokenizer, quantization_method = \"f16\")\n", + "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, quantization_method = \"f16\", token = \"\")\n", + "\n", + "# Save to q4_k_m GGUF\n", + "if False: model.save_pretrained_gguf(\"model\", tokenizer, quantization_method = \"q4_k_m\")\n", + "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, quantization_method = \"q4_k_m\", token = \"\")\n", + "\n", + "# Save to multiple GGUF options - much faster if you want multiple!\n", + "if False:\n", + " model.push_to_hub_gguf(\n", + " \"hf/model\", # Change hf to your username!\n", + " tokenizer,\n", + " quantization_method = [\"q4_k_m\", \"q8_0\", \"q5_k_m\",],\n", + " token = \"\", # Get a token at https://huggingface.co/settings/tokens\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kBvrJqNr6L9g" + }, + "source": [ + "Now, use the `model-unsloth.gguf` file or `model-unsloth-Q4_K_M.gguf` file in llama.cpp.\n", + "\n", + "And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n", + "\n", + "Some other links:\n", + "1. Train your own reasoning model - Llama GRPO notebook [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb)\n", + "2. Saving finetunes to Ollama. [Free notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb)\n", + "3. Llama 3.2 Vision finetuning - Radiography use case. [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb)\n", + "6. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://docs.unsloth.ai/get-started/unsloth-notebooks)!\n", + "\n", + "

\n", + " \n", + " \n", + " \n", + "\n", + " Join Discord if you need help + ⭐️ Star us on Github ⭐️\n", + "
\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "019f19eb060545539da7628564caaf5e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "02d6ff36400543d196671dcb609a3554": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "035a4fa66ce8447fbbed4adfd986726b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "03ea102e7b20453b939c9e514f7a61d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "04c78130763c4dfc93a86c788a6e01e8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0526bd2f6b014475bc4279fb6ad44ab0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1a89be1d70a043a7ab316fe731a4af95", + "max": 7512, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_cfd8bbe5291845f6b886415ece13f7f7", + "value": 7512 + } + }, + "052dc013a23944ca8d1de2d0c7ab8141": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "06ad2446cdb64c4db134046da4cfd0a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "071495bad0e143ce8b5514d75ce5d435": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9dd47b04490548bbb03c204544cc022e", + "placeholder": "​", + "style": "IPY_MODEL_524a6d91d0924912b4ce4a2f4c87e76a", + "value": " 7.03M/7.03M [00:00<00:00, 44.7MB/s]" + } + }, + "08d6a0314e864384aa3e163fb2224c35": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0dd1d1bffd654340b8d21af1f0eab11d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0df17c7f00c949c7b8444b91754fa4dd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0dfde06ab9d54e6b8c563d380f4b6026": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c4df1f0a6cb34bca9a84cb734ffcd6c7", + "placeholder": "​", + "style": "IPY_MODEL_8a87d368abe3434c82f99fb23e717e5e", + "value": " 613/613 [00:00<00:00, 36.5kB/s]" + } + }, + "0e66ab8fb2384f09a303a034fbde6247": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f65529309064ff781015f38375a34ce": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "101d6d98e8bb4a1ab216ddac084314ec": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1105390239d846898e653a7bfd092ee6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "167f32a4f26a4466a3c9b94fb7aa34b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "16946595a9064832a233c889e01818c9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "16dee8a92a9e4fbd908eeda7193fc190": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "17f32b453e4841a383c210853a36da84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "18a1a798b3824b33b73f345520583d91": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1a89be1d70a043a7ab316fe731a4af95": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1ad8199be4dd49ea8a9638d3e53d3d74": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8409a21d6d594569aca7721796af52ba", + "max": 7031863, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c883d3a00bc542db88244a71b924e1f0", + "value": 7031863 + } + }, + "1c971ddb47764c6aa4f766c5abd8cd71": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1ccf810e39ea4fbcb81f465c3b6bd601": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "213d400d9c704b9aaed21ad5a91c91a7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d09a3f6c289c4608a8dfcf53157bf1b6", + "IPY_MODEL_1ad8199be4dd49ea8a9638d3e53d3d74", + "IPY_MODEL_071495bad0e143ce8b5514d75ce5d435" + ], + "layout": "IPY_MODEL_7d7b05e1f1f642acb8f5ba6afdf6d82f" + } + }, + "23cf64d2dff94432af9cf6a7f99744ae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "244e4cf841cd483b8e9e165c77c471f0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b47587f1ba944ad9850fcf555cdbcf12", + "placeholder": "​", + "style": "IPY_MODEL_82806a720dad4704ba224812d9018bc2", + "value": " 100000/100000 [00:02<00:00, 41148.75 examples/s]" + } + }, + "24f12af7e8664d72ae38b06a713294a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a7d2b0d1a685407fa0633eb56f3676db", + "placeholder": "​", + "style": "IPY_MODEL_7dbc6c270eff43569640b25e450601ab", + "value": " 117M/117M [00:00<00:00, 137MB/s]" + } + }, + "254584c067e146809ab71a7a85d7c7a4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "25ce0fb1aed24b0cb1419ac3761109f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9d15aa6b077044f4a7aba7c3d543c984", + "placeholder": "​", + "style": "IPY_MODEL_9a2c98e1c8d5491791e4d7a1390cf81f", + "value": "added_tokens.json: 100%" + } + }, + "25d1aaf77b304450877876f5c8bd38fb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "260322195cc9480aaeb1173315a93894": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_55f6a6b67a754ef89732a29531dcf34a", + "placeholder": "​", + "style": "IPY_MODEL_25d1aaf77b304450877876f5c8bd38fb", + "value": " 5.00G/5.00G [00:33<00:00, 181MB/s]" + } + }, + "27f38f1ed0a14f119ba6b8f48662f5b9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2853c5ef931d422aa5a7622bc2f61bb2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f15f91be9d1e47da8079d1d0d48f7842", + "placeholder": "​", + "style": "IPY_MODEL_17f32b453e4841a383c210853a36da84", + "value": "Loading checkpoint shards: 100%" + } + }, + "2983c2011c194d0b88383d3bbf356845": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "298be59fbb97424080819b7a16db2024": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b851f3f552ad41fe97e9d7b72ea71b1c", + "IPY_MODEL_963f33764e744624bd33a8252511c0f2", + "IPY_MODEL_5f9574f2a91742d6a11be231ae60b4a6" + ], + "layout": "IPY_MODEL_895dccba65a5482c94d4c74371412241" + } + }, + "2cd72f4bdc1b41a4a2894c553d335dc5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3032a337e20945198e3678f91c371bc9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e1d1a5a6b94741a4b22e254c504264c0", + "IPY_MODEL_63b67e250d8349f88ee276ef732168c9", + "IPY_MODEL_cae54a63540742c3bb7a0412a5c55add" + ], + "layout": "IPY_MODEL_ae9f5ce0eefc4a8a98c2599a0fbf890a" + } + }, + "31400dbf077840769de1cb7f63cec131": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3215fa6f5c0446b9b6b4a0d15dd33032": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bccfa4e6adfd40d3afff51f5ab038422", + "placeholder": "​", + "style": "IPY_MODEL_d6292ba7c64e4e3f9baf68905657ec56", + "value": " 100000/100000 [00:07<00:00, 16491.66 examples/s]" + } + }, + "32b193ccdb024d5993e42b4e0efd1631": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3371554f74674d2da66fc8c794822c24": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "38db9cffac4a42759c3c1be2e5a4dd3f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3973d4960a584bcebe080796c99104f1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "39c1e706d28a46218940e01190b2843a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3ae2a86a77a34f61a204bd1355dda8b3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3b74d36ab9974076b06ffe7b545d582f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3c58773e281144f39081dc3414f9f4c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_71b65997837d405ebc3e92cb8b498faa", + "max": 2776833, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_bf4e7a04363d477cb36586a0fbcfc64e", + "value": 2776833 + } + }, + "413290be48d2410c93e2da6848affa39": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "41c627c3ffb144eca3cd45f490e61a30": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d67e9d729bee4325a5a01a90aecb52d0", + "placeholder": "​", + "style": "IPY_MODEL_7756a42c820745389107246cb6a35072", + "value": "model-00001-of-00002.safetensors: 100%" + } + }, + "44fdce605bf54fba80f81524461259bf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4949d26418b944f0a7d4c8e69a49f3f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_edf75f353d9645ba8796e2652d636166", + "IPY_MODEL_b6c160bf3d3043f8840ab4b7431301ca", + "IPY_MODEL_cacb47001cd341abae219b526401616d" + ], + "layout": "IPY_MODEL_1c971ddb47764c6aa4f766c5abd8cd71" + } + }, + "4d78f66177ca43f6b1efa5588078c0e4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_553a899890fb42348e5447ed43d5fd9b", + "placeholder": "​", + "style": "IPY_MODEL_b985370c39e64ea0943e1edda65a6bed", + "value": "tokenizer_config.json: 100%" + } + }, + "4e38a0d4d6ca440388951805ce4b6104": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4f5037bad4c849f19ccfd7b0f57f6721": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d69b25177b214cfb877907376763b4e6", + "IPY_MODEL_de85541764644bd685f7a950115c2544", + "IPY_MODEL_5808b66051e943c7ae58cf2dfebf2b41" + ], + "layout": "IPY_MODEL_5ee0f800a0934bc596a2b8249709c6e9" + } + }, + "524a6d91d0924912b4ce4a2f4c87e76a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "52a3d29efbd04ec7858876606b31465c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5341de66236b459ca83a154fb65e0596": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0dd1d1bffd654340b8d21af1f0eab11d", + "max": 613, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_f16ef7e24cd64841a3e2d8988d286c58", + "value": 613 + } + }, + "553a899890fb42348e5447ed43d5fd9b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "55aec8e495624b2faa5492e612807dbb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2cd72f4bdc1b41a4a2894c553d335dc5", + "placeholder": "​", + "style": "IPY_MODEL_9e31bce1bd2c4cf48724f9f5d9c5282f", + "value": " 7.51k/7.51k [00:00<00:00, 420kB/s]" + } + }, + "55f6a6b67a754ef89732a29531dcf34a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "56146e26a4054d87a843ace9758b7100": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "57c9f679f77241d0b6f9bf794dad2b9b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "5808b66051e943c7ae58cf2dfebf2b41": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6f98b1e08f954e82adc4565a0b7556c1", + "placeholder": "​", + "style": "IPY_MODEL_a263b7c2a34e41dabc94d78b99835ad8", + "value": " 100000/100000 [00:52<00:00, 1874.99 examples/s]" + } + }, + "594a5e73c6d24e8a8f98400c2ee99636": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_38db9cffac4a42759c3c1be2e5a4dd3f", + "placeholder": "​", + "style": "IPY_MODEL_32b193ccdb024d5993e42b4e0efd1631", + "value": " 982/982 [00:00<00:00, 20.3kB/s]" + } + }, + "5998b125c6a64ef886fc51d745b5301d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e2a1b3b26e84490c99b305d4a651cda4", + "IPY_MODEL_5341de66236b459ca83a154fb65e0596", + "IPY_MODEL_0dfde06ab9d54e6b8c563d380f4b6026" + ], + "layout": "IPY_MODEL_27f38f1ed0a14f119ba6b8f48662f5b9" + } + }, + "5d053c12d1ae427fa34c553d4759e25d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5d97ec03ac9544409f51c2ae1759dc8b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5ee0f800a0934bc596a2b8249709c6e9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f63b5e75daf44ada6af8dd6922a58ce": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b2def88c44ee4b608faaf38148b30c18", + "placeholder": "​", + "style": "IPY_MODEL_08d6a0314e864384aa3e163fb2224c35", + "value": " 1.67M/1.67M [00:00<00:00, 34.6MB/s]" + } + }, + "5f9574f2a91742d6a11be231ae60b4a6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_035a4fa66ce8447fbbed4adfd986726b", + "placeholder": "​", + "style": "IPY_MODEL_16dee8a92a9e4fbd908eeda7193fc190", + "value": " 210k/210k [00:00<00:00, 3.13MB/s]" + } + }, + "6178ea883ce546fd9f433d5a683d2d18": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_99c57c528f2d4c6284022efba3b9e81d", + "max": 4998641583, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1ccf810e39ea4fbcb81f465c3b6bd601", + "value": 4998641107 + } + }, + "617973da3a3f4b1f86729365d12f9875": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "63b67e250d8349f88ee276ef732168c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f65529309064ff781015f38375a34ce", + "max": 4933695760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_7350d931506e4e99b55a03751c453515", + "value": 4933695290 + } + }, + "6765cb1a549f45e998ccb19139e2e8a3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c2d9520cf1134cc182c2612bfd912875", + "IPY_MODEL_a53fab9c72f6468aaffd5bab3765af49", + "IPY_MODEL_244e4cf841cd483b8e9e165c77c471f0" + ], + "layout": "IPY_MODEL_8bbeb59a93de469682466e54bc6c35a7" + } + }, + "6b4a5dee5e3143a9b69dede345b7187d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b81f7175b224cb4a4320b33d969c146": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_41c627c3ffb144eca3cd45f490e61a30", + "IPY_MODEL_6178ea883ce546fd9f433d5a683d2d18", + "IPY_MODEL_260322195cc9480aaeb1173315a93894" + ], + "layout": "IPY_MODEL_c69c5e05efc8464f86c662b9ae931538" + } + }, + "6cbf4a502296486d9b0d8588dc123732": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6db5bed4b1c4478c97e07a7ab32d1845": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6e2018997fc545d8bb37c862fff0c751": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6f98b1e08f954e82adc4565a0b7556c1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "709b591f878747ee9a22c01bb8fa2c74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7135057d6cb84a77a04a8db3c6c3f336": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71b65997837d405ebc3e92cb8b498faa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "734f1dd7724d49f5a196552da6abf192": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b00ce2847b33492b9eba44d065ea7f06", + "placeholder": "​", + "style": "IPY_MODEL_7ba890d72f4245cb8f760f252ff80caa", + "value": "merges.txt: 100%" + } + }, + "7350d931506e4e99b55a03751c453515": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7382f0c83e7b44d3a19bbfda0d3723ee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "760c42f3b2054820acf6b1b6da11356a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "76a9c48d89e44cfc9d30cdf3b56a4f74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7756a42c820745389107246cb6a35072": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "789512128fa84d3481a086db62c44c37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e658cae26b3f457882ef5fe90309ed38", + "placeholder": "​", + "style": "IPY_MODEL_4e38a0d4d6ca440388951805ce4b6104", + "value": " 2/2 [00:53<00:00, 26.72s/it]" + } + }, + "7942f894ea8f4268b9c2b6ad83b2f279": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9c40f79c7efb4b758c6af85ba105fe4c", + "max": 2, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_413290be48d2410c93e2da6848affa39", + "value": 2 + } + }, + "7972d106d5414a218441942913aa3e3c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_167f32a4f26a4466a3c9b94fb7aa34b2", + "placeholder": "​", + "style": "IPY_MODEL_d5682db4b5cd4bb4abdb6ff7258caefa", + "value": " 2.78M/2.78M [00:00<00:00, 11.2MB/s]" + } + }, + "79ce9732a3af4920a64598318a491398": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7a285248d8fa4378860cd5e899613057": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ba5b4ef26bc14dadbd444c7a039a2995", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_18a1a798b3824b33b73f345520583d91", + "value": 100000 + } + }, + "7b2494ece7db465ca088670a7f8ffbef": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7ba890d72f4245cb8f760f252ff80caa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7cbc2991535b440b97e02a9df10b306a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7d7b05e1f1f642acb8f5ba6afdf6d82f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7dbc6c270eff43569640b25e450601ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "815aec1656224eacabc076618f7f067a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4d78f66177ca43f6b1efa5588078c0e4", + "IPY_MODEL_0526bd2f6b014475bc4279fb6ad44ab0", + "IPY_MODEL_55aec8e495624b2faa5492e612807dbb" + ], + "layout": "IPY_MODEL_e6eccabf31e545a2b06c7650c9781d05" + } + }, + "82806a720dad4704ba224812d9018bc2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8409a21d6d594569aca7721796af52ba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8892267f311b4cf0a5267ce193c5a646": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fa44d5de08014ac48013167025a3de1b", + "max": 2, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c9f72c0ebbee4aad8023b2082ce59f34", + "value": 2 + } + }, + "89520764d8ef4805805055bebd21273e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_56146e26a4054d87a843ace9758b7100", + "placeholder": "​", + "style": "IPY_MODEL_02d6ff36400543d196671dcb609a3554", + "value": " 632/632 [00:00<00:00, 45.3kB/s]" + } + }, + "895dccba65a5482c94d4c74371412241": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8a87d368abe3434c82f99fb23e717e5e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8b1c036ade754137a675d55d6ad12e2c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6cbf4a502296486d9b0d8588dc123732", + "placeholder": "​", + "style": "IPY_MODEL_7382f0c83e7b44d3a19bbfda0d3723ee", + "value": " 100000/100000 [04:03<00:00, 425.91 examples/s]" + } + }, + "8bbeb59a93de469682466e54bc6c35a7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8bcfc8c1bd6c474fabac2dfff5ca829d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_23cf64d2dff94432af9cf6a7f99744ae", + "placeholder": "​", + "style": "IPY_MODEL_bfef7ae67a7b43ed816246be0ed19a79", + "value": "Map: 100%" + } + }, + "8c6346ded74b4614871353d09b685ab3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7cbc2991535b440b97e02a9df10b306a", + "placeholder": "​", + "style": "IPY_MODEL_ac9c54ff279a4080b7485d669989db6d", + "value": "Map (num_proc=4): 100%" + } + }, + "8cf481de1931406e97951a2adfbdeee0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_101d6d98e8bb4a1ab216ddac084314ec", + "max": 116531415, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_617973da3a3f4b1f86729365d12f9875", + "value": 116531404 + } + }, + "8dd6e110704f40c995b9f0f6951cbb74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8f9648d8465c414ea4699a63fbf936a9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8fe160b637b24a74a4a97bdd9b1858d1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "93ccb0dfad4e4d60a7e7f42402bff80d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8c6346ded74b4614871353d09b685ab3", + "IPY_MODEL_7a285248d8fa4378860cd5e899613057", + "IPY_MODEL_8b1c036ade754137a675d55d6ad12e2c" + ], + "layout": "IPY_MODEL_b9304f02c4774b6eb161da6fa6b009ff" + } + }, + "963f33764e744624bd33a8252511c0f2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7135057d6cb84a77a04a8db3c6c3f336", + "max": 210129, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a12e1cf419bd446b9823fc924431cae4", + "value": 210129 + } + }, + "97359398bd724bf1acf5c4370bb7eaa2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "975b0754e56546d4bd7103b4f28a37c5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e38e2b0f246d4b369487fd405be38e15", + "placeholder": "​", + "style": "IPY_MODEL_6b4a5dee5e3143a9b69dede345b7187d", + "value": " 100000/100000 [00:27<00:00, 4638.49 examples/s]" + } + }, + "994b86b8732346de8b0789f366068549": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "99c57c528f2d4c6284022efba3b9e81d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9a2c98e1c8d5491791e4d7a1390cf81f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9b816b35709d4b4fbedd183e6a7aae5a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9c40f79c7efb4b758c6af85ba105fe4c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9d15aa6b077044f4a7aba7c3d543c984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9dd47b04490548bbb03c204544cc022e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9e31bce1bd2c4cf48724f9f5d9c5282f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9e7f78f4d932445daecd09dcf7b37859": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_994b86b8732346de8b0789f366068549", + "placeholder": "​", + "style": "IPY_MODEL_3b74d36ab9974076b06ffe7b545d582f", + "value": "Downloading shards: 100%" + } + }, + "a12e1cf419bd446b9823fc924431cae4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a263b7c2a34e41dabc94d78b99835ad8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a2951310cc79437a8dac88cadd9ed1e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_254584c067e146809ab71a7a85d7c7a4", + "placeholder": "​", + "style": "IPY_MODEL_44fdce605bf54fba80f81524461259bf", + "value": "train-00000-of-00001.parquet: 100%" + } + }, + "a53fab9c72f6468aaffd5bab3765af49": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8fe160b637b24a74a4a97bdd9b1858d1", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_57c9f679f77241d0b6f9bf794dad2b9b", + "value": 100000 + } + }, + "a6baf9fd477342fe8474022ac8e7a6ed": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a7d2b0d1a685407fa0633eb56f3676db": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8f4a39bb9d74a9bb4138c8bed4186e5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ac9c54ff279a4080b7485d669989db6d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "acda49009d124138843b187c2aa9e410": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "adc2a364ac1b4901acbae39562402743": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae9f5ce0eefc4a8a98c2599a0fbf890a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b00ce2847b33492b9eba44d065ea7f06": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b1a3ff316c374f509898c21c8618dcc9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b27f191668cf4d1cbf9705fe51e3c3df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e2018997fc545d8bb37c862fff0c751", + "placeholder": "​", + "style": "IPY_MODEL_acda49009d124138843b187c2aa9e410", + "value": "README.md: 100%" + } + }, + "b2def88c44ee4b608faaf38148b30c18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b468a2d515f943979af3d1921cda5594": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e7cf601a9b6342cfb38376989e08c5a9", + "max": 632, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ca23b61a88bf4e609075619b9d1111ef", + "value": 632 + } + }, + "b47587f1ba944ad9850fcf555cdbcf12": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b608a884c04541ca88253a72d1d81cb6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_760c42f3b2054820acf6b1b6da11356a", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_cf74cf0282cd409498112783ef3068aa", + "value": 100000 + } + }, + "b6c160bf3d3043f8840ab4b7431301ca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8dd6e110704f40c995b9f0f6951cbb74", + "max": 266, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c2dd2c13d62d48e09a68373f10dcd95a", + "value": 266 + } + }, + "b718bba8164047ccbb046d3475fb0f12": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_25ce0fb1aed24b0cb1419ac3761109f7", + "IPY_MODEL_b468a2d515f943979af3d1921cda5594", + "IPY_MODEL_89520764d8ef4805805055bebd21273e" + ], + "layout": "IPY_MODEL_adc2a364ac1b4901acbae39562402743" + } + }, + "b83dcf4897f544678d85a072cfc8bcca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a6baf9fd477342fe8474022ac8e7a6ed", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3371554f74674d2da66fc8c794822c24", + "value": 100000 + } + }, + "b851f3f552ad41fe97e9d7b72ea71b1c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_39c1e706d28a46218940e01190b2843a", + "placeholder": "​", + "style": "IPY_MODEL_5d97ec03ac9544409f51c2ae1759dc8b", + "value": "model.safetensors.index.json: 100%" + } + }, + "b9304f02c4774b6eb161da6fa6b009ff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b985370c39e64ea0943e1edda65a6bed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ba0b2bd35ce44100be27bc749642e1cc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_16946595a9064832a233c889e01818c9", + "placeholder": "​", + "style": "IPY_MODEL_ebdb9c8af7be4743b2e3c1308bddc0a5", + "value": "Standardizing format: 100%" + } + }, + "ba5b4ef26bc14dadbd444c7a039a2995": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bbac5526b00745c982a175bd180cbd24": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bccfa4e6adfd40d3afff51f5ab038422": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bf4e7a04363d477cb36586a0fbcfc64e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "bfef7ae67a7b43ed816246be0ed19a79": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c186068d654b4ba3a97105956d52083e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c19fa4dea6524d37843cbf5786b26221": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fba4df061ec84f4abe521ee628c53b62", + "max": 982, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e0cf51730c35468ea3074be96df8b8a0", + "value": 982 + } + }, + "c2d9520cf1134cc182c2612bfd912875": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0df17c7f00c949c7b8444b91754fa4dd", + "placeholder": "​", + "style": "IPY_MODEL_a8f4a39bb9d74a9bb4138c8bed4186e5", + "value": "Generating train split: 100%" + } + }, + "c2dd2c13d62d48e09a68373f10dcd95a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c4df1f0a6cb34bca9a84cb734ffcd6c7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c69c5e05efc8464f86c662b9ae931538": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c883d3a00bc542db88244a71b924e1f0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c9c96e91dc6347e1b6566570e925e40f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2853c5ef931d422aa5a7622bc2f61bb2", + "IPY_MODEL_8892267f311b4cf0a5267ce193c5a646", + "IPY_MODEL_789512128fa84d3481a086db62c44c37" + ], + "layout": "IPY_MODEL_8f9648d8465c414ea4699a63fbf936a9" + } + }, + "c9f72c0ebbee4aad8023b2082ce59f34": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ca23b61a88bf4e609075619b9d1111ef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cacb47001cd341abae219b526401616d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_97359398bd724bf1acf5c4370bb7eaa2", + "placeholder": "​", + "style": "IPY_MODEL_06ad2446cdb64c4db134046da4cfd0a5", + "value": " 266/266 [00:00<00:00, 18.5kB/s]" + } + }, + "cae54a63540742c3bb7a0412a5c55add": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_04c78130763c4dfc93a86c788a6e01e8", + "placeholder": "​", + "style": "IPY_MODEL_e3a4437a1f8b4710a77983b82f0ebe30", + "value": " 4.93G/4.93G [01:06<00:00, 16.8MB/s]" + } + }, + "cf74cf0282cd409498112783ef3068aa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cfd8bbe5291845f6b886415ece13f7f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d09a3f6c289c4608a8dfcf53157bf1b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_019f19eb060545539da7628564caaf5e", + "placeholder": "​", + "style": "IPY_MODEL_3973d4960a584bcebe080796c99104f1", + "value": "tokenizer.json: 100%" + } + }, + "d5682db4b5cd4bb4abdb6ff7258caefa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d6292ba7c64e4e3f9baf68905657ec56": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d67e9d729bee4325a5a01a90aecb52d0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69b25177b214cfb877907376763b4e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_052dc013a23944ca8d1de2d0c7ab8141", + "placeholder": "​", + "style": "IPY_MODEL_52a3d29efbd04ec7858876606b31465c", + "value": "Map: 100%" + } + }, + "d6d14db2e4894ad18b899cafb148b871": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d71c0ead75c2411587677493ad59256b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d94c60b5a75249a894c761c7baf12ff4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "da6b6fcb41f74c73b0b3460ba6d4c7dd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9e7f78f4d932445daecd09dcf7b37859", + "IPY_MODEL_7942f894ea8f4268b9c2b6ad83b2f279", + "IPY_MODEL_dfed5a49746e413bbaa44aa82a61228c" + ], + "layout": "IPY_MODEL_6db5bed4b1c4478c97e07a7ab32d1845" + } + }, + "de85541764644bd685f7a950115c2544": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d94c60b5a75249a894c761c7baf12ff4", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e37375168ae64bc1a8c8c1451dc2c4b2", + "value": 100000 + } + }, + "dee9866d465e43c99e61f71b819d9994": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba0b2bd35ce44100be27bc749642e1cc", + "IPY_MODEL_b83dcf4897f544678d85a072cfc8bcca", + "IPY_MODEL_3215fa6f5c0446b9b6b4a0d15dd33032" + ], + "layout": "IPY_MODEL_79ce9732a3af4920a64598318a491398" + } + }, + "dfed5a49746e413bbaa44aa82a61228c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1105390239d846898e653a7bfd092ee6", + "placeholder": "​", + "style": "IPY_MODEL_9b816b35709d4b4fbedd183e6a7aae5a", + "value": " 2/2 [01:40<00:00, 53.30s/it]" + } + }, + "e0cf51730c35468ea3074be96df8b8a0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e1d1a5a6b94741a4b22e254c504264c0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d71c0ead75c2411587677493ad59256b", + "placeholder": "​", + "style": "IPY_MODEL_c186068d654b4ba3a97105956d52083e", + "value": "model-00002-of-00002.safetensors: 100%" + } + }, + "e2a1b3b26e84490c99b305d4a651cda4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7b2494ece7db465ca088670a7f8ffbef", + "placeholder": "​", + "style": "IPY_MODEL_03ea102e7b20453b939c9e514f7a61d5", + "value": "special_tokens_map.json: 100%" + } + }, + "e37375168ae64bc1a8c8c1451dc2c4b2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e38e2b0f246d4b369487fd405be38e15": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e3a4437a1f8b4710a77983b82f0ebe30": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e658cae26b3f457882ef5fe90309ed38": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e6eccabf31e545a2b06c7650c9781d05": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e7371eff6f4c458792abd1e05e147110": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b27f191668cf4d1cbf9705fe51e3c3df", + "IPY_MODEL_c19fa4dea6524d37843cbf5786b26221", + "IPY_MODEL_594a5e73c6d24e8a8f98400c2ee99636" + ], + "layout": "IPY_MODEL_709b591f878747ee9a22c01bb8fa2c74" + } + }, + "e7cf601a9b6342cfb38376989e08c5a9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ebdb9c8af7be4743b2e3c1308bddc0a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ec4c9d4fe69645d78243987a8c12d503": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_734f1dd7724d49f5a196552da6abf192", + "IPY_MODEL_fcea53819cc441ec907174ee79ccc8ee", + "IPY_MODEL_5f63b5e75daf44ada6af8dd6922a58ce" + ], + "layout": "IPY_MODEL_31400dbf077840769de1cb7f63cec131" + } + }, + "edf75f353d9645ba8796e2652d636166": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_76a9c48d89e44cfc9d30cdf3b56a4f74", + "placeholder": "​", + "style": "IPY_MODEL_2983c2011c194d0b88383d3bbf356845", + "value": "generation_config.json: 100%" + } + }, + "f15f91be9d1e47da8079d1d0d48f7842": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f16ef7e24cd64841a3e2d8988d286c58": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "f4a36fec7d7a41e49bf125cc5cc98b0f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a2951310cc79437a8dac88cadd9ed1e6", + "IPY_MODEL_8cf481de1931406e97951a2adfbdeee0", + "IPY_MODEL_24f12af7e8664d72ae38b06a713294a5" + ], + "layout": "IPY_MODEL_0e66ab8fb2384f09a303a034fbde6247" + } + }, + "f6bee8f11d694040ae78e6f8616070d0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f7646ab3a3ff4106b33ab158c0296ae7", + "IPY_MODEL_3c58773e281144f39081dc3414f9f4c9", + "IPY_MODEL_7972d106d5414a218441942913aa3e3c" + ], + "layout": "IPY_MODEL_b1a3ff316c374f509898c21c8618dcc9" + } + }, + "f7646ab3a3ff4106b33ab158c0296ae7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bbac5526b00745c982a175bd180cbd24", + "placeholder": "​", + "style": "IPY_MODEL_5d053c12d1ae427fa34c553d4759e25d", + "value": "vocab.json: 100%" + } + }, + "f93b699993824be7a005d54d96cd6361": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "fa44d5de08014ac48013167025a3de1b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fba4df061ec84f4abe521ee628c53b62": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fcea53819cc441ec907174ee79ccc8ee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d6d14db2e4894ad18b899cafb148b871", + "max": 1671853, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_f93b699993824be7a005d54d96cd6361", + "value": 1671853 + } + }, + "ffcb4731dff04faa8323dceae528bce3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8bcfc8c1bd6c474fabac2dfff5ca829d", + "IPY_MODEL_b608a884c04541ca88253a72d1d81cb6", + "IPY_MODEL_975b0754e56546d4bd7103b4f28a37c5" + ], + "layout": "IPY_MODEL_3ae2a86a77a34f61a204bd1355dda8b3" + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/Qwen2_5_Coder_(14B)_Conversational (2).ipynb b/notebooks/Qwen2_5_Coder_(14B)_Conversational (2).ipynb new file mode 100644 index 0000000..03bcf8d --- /dev/null +++ b/notebooks/Qwen2_5_Coder_(14B)_Conversational (2).ipynb @@ -0,0 +1,8668 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "1UdZgdMa6L9X" + }, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xieWpvmi6L9Z" + }, + "source": [ + "### News" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T5zA3wvi6L9Z" + }, + "source": [ + "### Installation" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "yxLJvyfz6L9Z" + }, + "outputs": [], + "source": [ + "%%capture\n", + "import os, re\n", + "if \"COLAB_\" not in \"\".join(os.environ.keys()):\n", + " !pip install unsloth\n", + "else:\n", + " # Do this only in Colab notebooks! Otherwise use pip install unsloth\n", + " import torch; v = re.match(r\"[0-9\\.]{3,}\", str(torch.__version__)).group(0)\n", + " xformers = \"xformers==\" + (\"0.0.32.post2\" if v == \"2.8.0\" else \"0.0.29.post3\")\n", + " !pip install --no-deps bitsandbytes accelerate {xformers} peft trl triton cut_cross_entropy\n", + " !pip install sentencepiece protobuf \"datasets>=3.4.1,<4.0.0\" \"huggingface_hub>=0.34.0\" hf_transfer\n", + " !pip install \"unsloth_zoo[base] @ git+https://github.com/unslothai/unsloth-zoo\"\n", + " !pip install \"unsloth[base] @ git+https://github.com/unslothai/unsloth\"\n", + "!pip install transformers==4.55.4" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hSZDfEpp6L9a" + }, + "source": [ + "### Unsloth" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 880, + "referenced_widgets": [ + "298be59fbb97424080819b7a16db2024", + "b851f3f552ad41fe97e9d7b72ea71b1c", + "963f33764e744624bd33a8252511c0f2", + "5f9574f2a91742d6a11be231ae60b4a6", + "895dccba65a5482c94d4c74371412241", + "39c1e706d28a46218940e01190b2843a", + "5d97ec03ac9544409f51c2ae1759dc8b", + "7135057d6cb84a77a04a8db3c6c3f336", + "a12e1cf419bd446b9823fc924431cae4", + "035a4fa66ce8447fbbed4adfd986726b", + "16dee8a92a9e4fbd908eeda7193fc190", + "da6b6fcb41f74c73b0b3460ba6d4c7dd", + "9e7f78f4d932445daecd09dcf7b37859", + "7942f894ea8f4268b9c2b6ad83b2f279", + "dfed5a49746e413bbaa44aa82a61228c", + "6db5bed4b1c4478c97e07a7ab32d1845", + "994b86b8732346de8b0789f366068549", + "3b74d36ab9974076b06ffe7b545d582f", + "9c40f79c7efb4b758c6af85ba105fe4c", + "413290be48d2410c93e2da6848affa39", + "1105390239d846898e653a7bfd092ee6", + "9b816b35709d4b4fbedd183e6a7aae5a", + "6b81f7175b224cb4a4320b33d969c146", + "41c627c3ffb144eca3cd45f490e61a30", + "6178ea883ce546fd9f433d5a683d2d18", + "260322195cc9480aaeb1173315a93894", + "c69c5e05efc8464f86c662b9ae931538", + "d67e9d729bee4325a5a01a90aecb52d0", + "7756a42c820745389107246cb6a35072", + "99c57c528f2d4c6284022efba3b9e81d", + "1ccf810e39ea4fbcb81f465c3b6bd601", + "55f6a6b67a754ef89732a29531dcf34a", + "25d1aaf77b304450877876f5c8bd38fb", + "3032a337e20945198e3678f91c371bc9", + "e1d1a5a6b94741a4b22e254c504264c0", + "63b67e250d8349f88ee276ef732168c9", + "cae54a63540742c3bb7a0412a5c55add", + "ae9f5ce0eefc4a8a98c2599a0fbf890a", + "d71c0ead75c2411587677493ad59256b", + "c186068d654b4ba3a97105956d52083e", + "0f65529309064ff781015f38375a34ce", + "7350d931506e4e99b55a03751c453515", + "04c78130763c4dfc93a86c788a6e01e8", + "e3a4437a1f8b4710a77983b82f0ebe30", + "c9c96e91dc6347e1b6566570e925e40f", + "2853c5ef931d422aa5a7622bc2f61bb2", + "8892267f311b4cf0a5267ce193c5a646", + "789512128fa84d3481a086db62c44c37", + "8f9648d8465c414ea4699a63fbf936a9", + "f15f91be9d1e47da8079d1d0d48f7842", + "17f32b453e4841a383c210853a36da84", + "fa44d5de08014ac48013167025a3de1b", + "c9f72c0ebbee4aad8023b2082ce59f34", + "e658cae26b3f457882ef5fe90309ed38", + "4e38a0d4d6ca440388951805ce4b6104", + "4949d26418b944f0a7d4c8e69a49f3f7", + "edf75f353d9645ba8796e2652d636166", + "b6c160bf3d3043f8840ab4b7431301ca", + "cacb47001cd341abae219b526401616d", + "1c971ddb47764c6aa4f766c5abd8cd71", + "76a9c48d89e44cfc9d30cdf3b56a4f74", + "2983c2011c194d0b88383d3bbf356845", + "8dd6e110704f40c995b9f0f6951cbb74", + "c2dd2c13d62d48e09a68373f10dcd95a", + "97359398bd724bf1acf5c4370bb7eaa2", + "06ad2446cdb64c4db134046da4cfd0a5", + "815aec1656224eacabc076618f7f067a", + "4d78f66177ca43f6b1efa5588078c0e4", + "0526bd2f6b014475bc4279fb6ad44ab0", + "55aec8e495624b2faa5492e612807dbb", + "e6eccabf31e545a2b06c7650c9781d05", + "553a899890fb42348e5447ed43d5fd9b", + "b985370c39e64ea0943e1edda65a6bed", + "1a89be1d70a043a7ab316fe731a4af95", + "cfd8bbe5291845f6b886415ece13f7f7", + "2cd72f4bdc1b41a4a2894c553d335dc5", + "9e31bce1bd2c4cf48724f9f5d9c5282f", + "f6bee8f11d694040ae78e6f8616070d0", + "f7646ab3a3ff4106b33ab158c0296ae7", + "3c58773e281144f39081dc3414f9f4c9", + "7972d106d5414a218441942913aa3e3c", + "b1a3ff316c374f509898c21c8618dcc9", + "bbac5526b00745c982a175bd180cbd24", + "5d053c12d1ae427fa34c553d4759e25d", + "71b65997837d405ebc3e92cb8b498faa", + "bf4e7a04363d477cb36586a0fbcfc64e", + "167f32a4f26a4466a3c9b94fb7aa34b2", + "d5682db4b5cd4bb4abdb6ff7258caefa", + "ec4c9d4fe69645d78243987a8c12d503", + "734f1dd7724d49f5a196552da6abf192", + "fcea53819cc441ec907174ee79ccc8ee", + "5f63b5e75daf44ada6af8dd6922a58ce", + "31400dbf077840769de1cb7f63cec131", + "b00ce2847b33492b9eba44d065ea7f06", + "7ba890d72f4245cb8f760f252ff80caa", + "d6d14db2e4894ad18b899cafb148b871", + "f93b699993824be7a005d54d96cd6361", + "b2def88c44ee4b608faaf38148b30c18", + "08d6a0314e864384aa3e163fb2224c35", + "b718bba8164047ccbb046d3475fb0f12", + "25ce0fb1aed24b0cb1419ac3761109f7", + "b468a2d515f943979af3d1921cda5594", + "89520764d8ef4805805055bebd21273e", + "adc2a364ac1b4901acbae39562402743", + "9d15aa6b077044f4a7aba7c3d543c984", + "9a2c98e1c8d5491791e4d7a1390cf81f", + "e7cf601a9b6342cfb38376989e08c5a9", + "ca23b61a88bf4e609075619b9d1111ef", + "56146e26a4054d87a843ace9758b7100", + "02d6ff36400543d196671dcb609a3554", + "5998b125c6a64ef886fc51d745b5301d", + "e2a1b3b26e84490c99b305d4a651cda4", + "5341de66236b459ca83a154fb65e0596", + "0dfde06ab9d54e6b8c563d380f4b6026", + "27f38f1ed0a14f119ba6b8f48662f5b9", + "7b2494ece7db465ca088670a7f8ffbef", + "03ea102e7b20453b939c9e514f7a61d5", + "0dd1d1bffd654340b8d21af1f0eab11d", + "f16ef7e24cd64841a3e2d8988d286c58", + "c4df1f0a6cb34bca9a84cb734ffcd6c7", + "8a87d368abe3434c82f99fb23e717e5e", + "213d400d9c704b9aaed21ad5a91c91a7", + "d09a3f6c289c4608a8dfcf53157bf1b6", + "1ad8199be4dd49ea8a9638d3e53d3d74", + "071495bad0e143ce8b5514d75ce5d435", + "7d7b05e1f1f642acb8f5ba6afdf6d82f", + "019f19eb060545539da7628564caaf5e", + "3973d4960a584bcebe080796c99104f1", + "8409a21d6d594569aca7721796af52ba", + "c883d3a00bc542db88244a71b924e1f0", + "9dd47b04490548bbb03c204544cc022e", + "524a6d91d0924912b4ce4a2f4c87e76a" + ] + }, + "id": "QmUBVEnvCDJv", + "outputId": "833593bc-378f-4726-ac6f-8bf57b0ee091" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "🦥 Unsloth Zoo will now patch everything to make training faster!\n", + "==((====))== Unsloth 2025.8.9: Fast Qwen2 patching. Transformers: 4.55.4.\n", + " \\\\ /| NVIDIA A100-SXM4-80GB. Num GPUs = 1. Max memory: 79.254 GB. Platform: Linux.\n", + "O^O/ \\_/ \\ Torch: 2.8.0+cu128. CUDA: 8.0. CUDA Toolkit: 12.8. Triton: 3.4.0\n", + "\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.32.post2. FA2 = False]\n", + " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", + "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Unsloth: unsloth/qwen2.5-72b-instruct-bnb-4bit can only handle sequence lengths of at most 32768.\n", + "But with kaiokendev's RoPE scaling of 1.465, it can be magically be extended to 48000!\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "dbc5546abfd948f19843459c132922f8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/9 [00:00 0 ! Suggested 8, 16, 32, 64, 128\n", + " target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", + " \"gate_proj\", \"up_proj\", \"down_proj\",],\n", + " lora_alpha = 16,\n", + " lora_dropout = 0, # Supports any, but = 0 is optimized\n", + " bias = \"none\", # Supports any, but = \"none\" is optimized\n", + " # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n", + " use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n", + " random_state = 3407,\n", + " use_rslora = False, # We support rank stabilized LoRA\n", + " loftq_config = None, # And LoftQ\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vITh0KVJ10qX" + }, + "source": [ + "\n", + "### Data Prep\n", + "We now use the `Qwen-2.5` format for conversation style finetunes. We use [Maxime Labonne's FineTome-100k](https://huggingface.co/datasets/mlabonne/FineTome-100k) dataset in ShareGPT style. But we convert it to HuggingFace's normal multiturn format `(\"role\", \"content\")` instead of `(\"from\", \"value\")`/ Qwen renders multi turn conversations like below:\n", + "\n", + "```\n", + "<|im_start|>system\n", + "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", + "<|im_start|>user\n", + "What is 2+2?<|im_end|>\n", + "<|im_start|>assistant\n", + "It's 4.<|im_end|>\n", + "\n", + "```\n", + "\n", + "We use our `get_chat_template` function to get the correct chat template. We support `zephyr, chatml, mistral, llama, alpaca, vicuna, vicuna_old, phi3, llama3` and more." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 10000 examples\n", + "Dataset({\n", + " features: ['conversation', 'metadata', 'original_example'],\n", + " num_rows: 10000\n", + "})\n", + "{'conversation': [{'content': 'You are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.', 'role': 'system'}, {'content': 'I\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\nElasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n\\n\\nCan you help me debug this systematically and explain your reasoning?', 'role': 'user'}, {'content': '', 'role': 'assistant'}], 'metadata': {'base_commit': '160cfef6e1118061fa66d333e8c2a572f5d0a815', 'dataset': 'princeton-nlp/SWE-bench', 'generation_timestamp': '2025-08-19T23:35:38.773997', 'has_patch': True, 'instance_id': 'DataDog__integrations-core-10093', 'original_index': 0, 'reasoning_length': 0, 'repo': 'DataDog/integrations-core', 'source': 'swe-bench'}, 'original_example': {'instance_id': 'DataDog__integrations-core-10093', 'patch': \"diff --git a/elastic/datadog_checks/elastic/elastic.py b/elastic/datadog_checks/elastic/elastic.py\\n--- a/elastic/datadog_checks/elastic/elastic.py\\n+++ b/elastic/datadog_checks/elastic/elastic.py\\n@@ -139,11 +139,17 @@ def _get_es_version(self):\\n try:\\n data = self._get_data(self._config.url, send_sc=False)\\n raw_version = data['version']['number']\\n+\\n self.set_metadata('version', raw_version)\\n # pre-release versions of elasticearch are suffixed with -rcX etc..\\n # peel that off so that the map below doesn't error out\\n raw_version = raw_version.split('-')[0]\\n version = [int(p) for p in raw_version.split('.')[0:3]]\\n+ if data['version'].get('distribution', '') == 'opensearch':\\n+ # Opensearch API is backwards compatible with ES 7.10.0\\n+ # https://opensearch.org/faq\\n+ self.log.debug('OpenSearch version %s detected', version)\\n+ version = [7, 10, 0]\\n except AuthenticationError:\\n raise\\n except Exception as e:\\n\", 'problem_statement': 'Elasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n', 'repo': 'DataDog/integrations-core'}}\n" + ] + } + ], + "source": [ + "from datasets import Dataset\n", + "import json\n", + "\n", + "file_path = \"data/swe_reasoning_dataset (3).jsonl\"\n", + "\n", + "# Load JSONL\n", + "data = []\n", + "with open(file_path, \"r\", encoding=\"utf-8\") as f:\n", + " for line in f:\n", + " data.append(json.loads(line))\n", + "\n", + "print(f\"Loaded {len(data)} examples\")\n", + "\n", + "# Convert to HuggingFace Dataset\n", + "dataset = Dataset.from_list(data)\n", + "\n", + "print(dataset)\n", + "print(dataset[0]) # Show first example\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 141, + "referenced_widgets": [ + "e7371eff6f4c458792abd1e05e147110", + "b27f191668cf4d1cbf9705fe51e3c3df", + "c19fa4dea6524d37843cbf5786b26221", + "594a5e73c6d24e8a8f98400c2ee99636", + "709b591f878747ee9a22c01bb8fa2c74", + "6e2018997fc545d8bb37c862fff0c751", + "acda49009d124138843b187c2aa9e410", + "fba4df061ec84f4abe521ee628c53b62", + "e0cf51730c35468ea3074be96df8b8a0", + "38db9cffac4a42759c3c1be2e5a4dd3f", + "32b193ccdb024d5993e42b4e0efd1631", + "f4a36fec7d7a41e49bf125cc5cc98b0f", + "a2951310cc79437a8dac88cadd9ed1e6", + "8cf481de1931406e97951a2adfbdeee0", + "24f12af7e8664d72ae38b06a713294a5", + "0e66ab8fb2384f09a303a034fbde6247", + "254584c067e146809ab71a7a85d7c7a4", + "44fdce605bf54fba80f81524461259bf", + "101d6d98e8bb4a1ab216ddac084314ec", + "617973da3a3f4b1f86729365d12f9875", + "a7d2b0d1a685407fa0633eb56f3676db", + "7dbc6c270eff43569640b25e450601ab", + "6765cb1a549f45e998ccb19139e2e8a3", + "c2d9520cf1134cc182c2612bfd912875", + "a53fab9c72f6468aaffd5bab3765af49", + "244e4cf841cd483b8e9e165c77c471f0", + "8bbeb59a93de469682466e54bc6c35a7", + "0df17c7f00c949c7b8444b91754fa4dd", + "a8f4a39bb9d74a9bb4138c8bed4186e5", + "8fe160b637b24a74a4a97bdd9b1858d1", + "57c9f679f77241d0b6f9bf794dad2b9b", + "b47587f1ba944ad9850fcf555cdbcf12", + "82806a720dad4704ba224812d9018bc2" + ] + }, + "id": "LjY75GoYUCB8", + "outputId": "5cf90520-4c68-4844-c050-b72a075f3ee6" + }, + "outputs": [], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template = \"qwen-2.5\",\n", + ")\n", + "\n", + "def formatting_prompts_func(examples):\n", + " convos = examples[\"conversation\"]\n", + " texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for convo in convos]\n", + " return { \"text\" : texts, }\n", + "pass\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K9CBpiISFa6C" + }, + "source": [ + "We now use `standardize_sharegpt` to convert ShareGPT style datasets into HuggingFace's generic format. This changes the dataset from looking like:\n", + "```\n", + "{\"from\": \"system\", \"value\": \"You are an assistant\"}\n", + "{\"from\": \"human\", \"value\": \"What is 2+2?\"}\n", + "{\"from\": \"gpt\", \"value\": \"It's 4.\"}\n", + "```\n", + "to\n", + "```\n", + "{\"role\": \"system\", \"content\": \"You are an assistant\"}\n", + "{\"role\": \"user\", \"content\": \"What is 2+2?\"}\n", + "{\"role\": \"assistant\", \"content\": \"It's 4.\"}\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 157, + "referenced_widgets": [ + "dee9866d465e43c99e61f71b819d9994", + "ba0b2bd35ce44100be27bc749642e1cc", + "b83dcf4897f544678d85a072cfc8bcca", + "3215fa6f5c0446b9b6b4a0d15dd33032", + "79ce9732a3af4920a64598318a491398", + "16946595a9064832a233c889e01818c9", + "ebdb9c8af7be4743b2e3c1308bddc0a5", + "a6baf9fd477342fe8474022ac8e7a6ed", + "3371554f74674d2da66fc8c794822c24", + "bccfa4e6adfd40d3afff51f5ab038422", + "d6292ba7c64e4e3f9baf68905657ec56", + "ffcb4731dff04faa8323dceae528bce3", + "8bcfc8c1bd6c474fabac2dfff5ca829d", + "b608a884c04541ca88253a72d1d81cb6", + "975b0754e56546d4bd7103b4f28a37c5", + "3ae2a86a77a34f61a204bd1355dda8b3", + "23cf64d2dff94432af9cf6a7f99744ae", + "bfef7ae67a7b43ed816246be0ed19a79", + "760c42f3b2054820acf6b1b6da11356a", + "cf74cf0282cd409498112783ef3068aa", + "e38e2b0f246d4b369487fd405be38e15", + "6b4a5dee5e3143a9b69dede345b7187d" + ] + }, + "id": "oPXzJZzHEgXe", + "outputId": "a1d12c39-303d-4ffc-83e9-3e4dd35cf172" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1110cc5449b44a60a688b90331bd35ad", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map: 0%| | 0/10000 [00:00system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n<|im_start|>user\\nHow do astronomers determine the original wavelength of light emitted by a celestial body at rest, which is necessary for measuring its speed using the Doppler effect?<|im_end|>\\n<|im_start|>assistant\\nAstronomers make use of the unique spectral fingerprints of elements found in stars. These elements emit and absorb light at specific, known wavelengths, forming an absorption spectrum. By analyzing the light received from distant stars and comparing it to the laboratory-measured spectra of these elements, astronomers can identify the shifts in these wavelengths due to the Doppler effect. The observed shift tells them the extent to which the light has been redshifted or blueshifted, thereby allowing them to calculate the speed of the star along the line of sight relative to Earth.<|im_end|>\\n'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset[5][\"text\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "idAEIeSQ3xdS" + }, + "source": [ + "\n", + "### Train the model\n", + "Now let's train our model. We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support TRL's `DPOTrainer`!\n", + "\n", + "The trainer includes our **gradient accumulation bug fix**. Read more about it here: [Blog post](https://unsloth.ai/blog/gradient)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 67, + "referenced_widgets": [ + "93ccb0dfad4e4d60a7e7f42402bff80d", + "8c6346ded74b4614871353d09b685ab3", + "7a285248d8fa4378860cd5e899613057", + "8b1c036ade754137a675d55d6ad12e2c", + "b9304f02c4774b6eb161da6fa6b009ff", + "7cbc2991535b440b97e02a9df10b306a", + "ac9c54ff279a4080b7485d669989db6d", + "ba5b4ef26bc14dadbd444c7a039a2995", + "18a1a798b3824b33b73f345520583d91", + "6cbf4a502296486d9b0d8588dc123732", + "7382f0c83e7b44d3a19bbfda0d3723ee" + ] + }, + "id": "95_Nn-89DhsL", + "outputId": "316cba0c-e677-47d5-c678-5b8398facc1e" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8a541f2dcfe840438d4c61f846417970", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Unsloth: Tokenizing [\"text\"] (num_proc=2): 0%| | 0/10000 [00:00user\\n\",\n", + " response_part = \"<|im_start|>assistant\\n\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Dv1NBUozV78l" + }, + "source": [ + "We verify masking is actually done:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 175 + }, + "id": "LtsMVtlkUhja", + "outputId": "cadeca7a-43d9-4a4b-bc80-c25cff082e2e" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'<|im_start|>system\\nYou are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.<|im_end|>\\n<|im_start|>user\\nI\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\n[apache] fix metric collection\\n### What does this PR do?\\r\\n\\r\\nThis collects the correct data for `apache.net.bytes_per_s` and `apache.net.request_per_s`.\\r\\n\\r\\nhttps://www.datadoghq.com/blog/collect-apache-performance-metrics\\r\\nhttps://wiki.opennms.org/wiki/Monitoring_Apache_with_the_HTTP_collector\\r\\nhttps://httpd.apache.org/docs/current/mod/mod_status.html\\r\\n\\r\\n### Motivation\\r\\n\\r\\nCustomer encountered bug\\n\\n\\nCan you help me debug this systematically and explain your reasoning?<|im_end|>\\n<|im_start|>assistant\\n## INITIAL PROBLEM ASSESSMENT\\n\\nThe bug report indicates that there was an issue with metric collection for Apache performance metrics, specifically `apache.net.bytes_per_s` and `apache.net.request_per_s`. This suggests that the application was not reporting accurate or expected values for these metrics, which could lead to misinterpretation of server performance by users relying on this data.\\n\\nSymptoms might include:\\n- Users not seeing expected values in monitoring dashboards.\\n- Discrepancies in data reported over time, potentially leading to false conclusions about server load and performance.\\n- Customer complaints about the monitoring system\\'s reliability.\\n\\n## HYPOTHESIS FORMATION\\n\\n1. **Incorrect Metric Mapping**: The metrics `apache.net.bytes_per_s` and `apache.net.request_per_s` might not be correctly mapped in the code, leading to values not being collected or reported as intended.\\n \\n2. **Improper Metric Calculation**: There may be a logical error in the way metrics are calculated or aggregated, such as not accounting for rates correctly.\\n\\n3. **Configuration Issues**: There could be an issue with how Apache is configured, potentially affecting what metrics are exposed and collected by the monitoring tool.\\n\\n**Ranked by likelihood**: \\n1. Incorrect Metric Mapping \\n2. Improper Metric Calculation \\n3. Configuration Issues\\n\\n## SYSTEMATIC INVESTIGATION\\n\\n1. **Review Code for Metric Mapping**:\\n - Examine the sections of code where metrics are defined and mapped to ensure that `apache.net.bytes_per_s` and `apache.net.request_per_s` are correctly included in the collection logic.\\n - Look for any recent changes in the code that might have affected the mapping.\\n\\n2. **Check Metric Calculation Logic**:\\n - Investigate how the metrics are being calculated. Look for functions or methods that aggregate or compute the values for these metrics.\\n - Confirm if the metrics are being treated as rates (per second) and check the logic for calculating rates.\\n\\n3. **Analyze Apache Configuration**:\\n - Validate the Apache configuration to ensure that the necessary modules (like `mod_status`) are enabled and that the URLs used for metric collection are correct.\\n - Check if the Apache server is correctly exposing the expected data needed for the metrics.\\n\\n## ROOT CAUSE IDENTIFICATION\\n\\nUpon reviewing the code changes in the provided patch, it becomes evident that the root cause was related to the way metrics were structured and collected. The initial code had the metrics defined in one section but did not correctly leverage them in the main collection logic. The patch introduced a dedicated section for rates (`RATES`), clearly defining how to collect `apache.net.bytes_per_s` and `apache.net.request_per_s` as rates. \\n\\n## SOLUTION DESIGN\\n\\nThe solution was designed to:\\n- Explicitly separate gauge metrics from rate metrics, improving clarity and correctness in how metrics are collected and reported.\\n- Ensure that the correct metrics are calculated and reported in the way that aligns with the expected definitions used by the monitoring system.\\n\\nThis approach was preferred over simply fixing the existing code because it allowed for better organization and future scalability. The separation of metric types makes it easier to manage and extend.\\n\\n## IMPLEMENTATION REASONING\\n\\n1. **Version Update**:\\n ```python\\n __version__ = \"1.1.2\"\\n ```\\n - Incrementing the version indicates a significant change in functionality that should be noted by users.\\n\\n2. **Defining RATES**:\\n ```python\\n RATES = {\\n \\'Total kBytes\\': \\'apache.net.bytes_per_s\\',\\n \\'Total Accesses\\': \\'apache.net.request_per_s\\'\\n }\\n ```\\n - Clearly defining the metrics that should be treated as rates ensures that the correct calculations are applied.\\n\\n3. **Updating the Metric Collection Logic**:\\n ```python\\n if metric in self.RATES:\\n metric_count += 1\\n metric_name = self.RATES[metric]\\n self.rate(metric_name, value, tags=tags)\\n ```\\n - This change ensures that when a metric is recognized as a rate, it is processed correctly using the `self.rate()` method, allowing for proper rate calculation.\\n\\n## VERIFICATION STRATEGY\\n\\nTo verify the fix:\\n- Test the updated metrics in a local or staging environment to ensure they are correctly reported as expected.\\n- Compare the output against expected values based on known server load and traffic patterns.\\n- Conduct load tests to ensure metrics react correctly under different conditions.\\n- Check edge cases, such as very low traffic scenarios, to ensure metrics do not report erroneous values like zero rates.\\n\\n## PREVENTION & LEARNING\\n\\nTo prevent similar issues in the future:\\n- Implement unit tests for metric collection to catch incorrect mappings or calculations before deployment.\\n- Adopt a metric design pattern that clearly separates different types of metrics (e.g., gauges vs. rates) and ensure that new metrics adhere to this structure.\\n- Regularly review and refactor code related to metrics to maintain clarity and correctness as the application evolves. \\n\\nThis case highlights the importance of clear metric definitions and the need for structured code when dealing with performance monitoring, which can significantly affect user trust and system reliability.<|im_end|>\\n'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode(trainer.train_dataset[5][\"input_ids\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 140 + }, + "id": "_rD6fl8EUxnG", + "outputId": "b1da5770-3de6-48d5-c4dc-890075d0b6b0" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "' ## INITIAL PROBLEM ASSESSMENT\\n\\nThe bug report indicates that there was an issue with metric collection for Apache performance metrics, specifically `apache.net.bytes_per_s` and `apache.net.request_per_s`. This suggests that the application was not reporting accurate or expected values for these metrics, which could lead to misinterpretation of server performance by users relying on this data.\\n\\nSymptoms might include:\\n- Users not seeing expected values in monitoring dashboards.\\n- Discrepancies in data reported over time, potentially leading to false conclusions about server load and performance.\\n- Customer complaints about the monitoring system\\'s reliability.\\n\\n## HYPOTHESIS FORMATION\\n\\n1. **Incorrect Metric Mapping**: The metrics `apache.net.bytes_per_s` and `apache.net.request_per_s` might not be correctly mapped in the code, leading to values not being collected or reported as intended.\\n \\n2. **Improper Metric Calculation**: There may be a logical error in the way metrics are calculated or aggregated, such as not accounting for rates correctly.\\n\\n3. **Configuration Issues**: There could be an issue with how Apache is configured, potentially affecting what metrics are exposed and collected by the monitoring tool.\\n\\n**Ranked by likelihood**: \\n1. Incorrect Metric Mapping \\n2. Improper Metric Calculation \\n3. Configuration Issues\\n\\n## SYSTEMATIC INVESTIGATION\\n\\n1. **Review Code for Metric Mapping**:\\n - Examine the sections of code where metrics are defined and mapped to ensure that `apache.net.bytes_per_s` and `apache.net.request_per_s` are correctly included in the collection logic.\\n - Look for any recent changes in the code that might have affected the mapping.\\n\\n2. **Check Metric Calculation Logic**:\\n - Investigate how the metrics are being calculated. Look for functions or methods that aggregate or compute the values for these metrics.\\n - Confirm if the metrics are being treated as rates (per second) and check the logic for calculating rates.\\n\\n3. **Analyze Apache Configuration**:\\n - Validate the Apache configuration to ensure that the necessary modules (like `mod_status`) are enabled and that the URLs used for metric collection are correct.\\n - Check if the Apache server is correctly exposing the expected data needed for the metrics.\\n\\n## ROOT CAUSE IDENTIFICATION\\n\\nUpon reviewing the code changes in the provided patch, it becomes evident that the root cause was related to the way metrics were structured and collected. The initial code had the metrics defined in one section but did not correctly leverage them in the main collection logic. The patch introduced a dedicated section for rates (`RATES`), clearly defining how to collect `apache.net.bytes_per_s` and `apache.net.request_per_s` as rates. \\n\\n## SOLUTION DESIGN\\n\\nThe solution was designed to:\\n- Explicitly separate gauge metrics from rate metrics, improving clarity and correctness in how metrics are collected and reported.\\n- Ensure that the correct metrics are calculated and reported in the way that aligns with the expected definitions used by the monitoring system.\\n\\nThis approach was preferred over simply fixing the existing code because it allowed for better organization and future scalability. The separation of metric types makes it easier to manage and extend.\\n\\n## IMPLEMENTATION REASONING\\n\\n1. **Version Update**:\\n ```python\\n __version__ = \"1.1.2\"\\n ```\\n - Incrementing the version indicates a significant change in functionality that should be noted by users.\\n\\n2. **Defining RATES**:\\n ```python\\n RATES = {\\n \\'Total kBytes\\': \\'apache.net.bytes_per_s\\',\\n \\'Total Accesses\\': \\'apache.net.request_per_s\\'\\n }\\n ```\\n - Clearly defining the metrics that should be treated as rates ensures that the correct calculations are applied.\\n\\n3. **Updating the Metric Collection Logic**:\\n ```python\\n if metric in self.RATES:\\n metric_count += 1\\n metric_name = self.RATES[metric]\\n self.rate(metric_name, value, tags=tags)\\n ```\\n - This change ensures that when a metric is recognized as a rate, it is processed correctly using the `self.rate()` method, allowing for proper rate calculation.\\n\\n## VERIFICATION STRATEGY\\n\\nTo verify the fix:\\n- Test the updated metrics in a local or staging environment to ensure they are correctly reported as expected.\\n- Compare the output against expected values based on known server load and traffic patterns.\\n- Conduct load tests to ensure metrics react correctly under different conditions.\\n- Check edge cases, such as very low traffic scenarios, to ensure metrics do not report erroneous values like zero rates.\\n\\n## PREVENTION & LEARNING\\n\\nTo prevent similar issues in the future:\\n- Implement unit tests for metric collection to catch incorrect mappings or calculations before deployment.\\n- Adopt a metric design pattern that clearly separates different types of metrics (e.g., gauges vs. rates) and ensure that new metrics adhere to this structure.\\n- Regularly review and refactor code related to metrics to maintain clarity and correctness as the application evolves. \\n\\nThis case highlights the importance of clear metric definitions and the need for structured code when dealing with performance monitoring, which can significantly affect user trust and system reliability.<|im_end|>\\n'" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "space = tokenizer(\" \", add_special_tokens = False).input_ids[0]\n", + "tokenizer.decode([space if x == -100 else x for x in trainer.train_dataset[5][\"labels\"]])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3enWUM0jV-jV" + }, + "source": [ + "We can see the System and Instruction prompts are successfully masked!" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2ejIt2xSNKKp", + "outputId": "faf37c55-a0ad-41a3-aeed-55ee7d7070e0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPU = NVIDIA A100-SXM4-80GB. Max memory = 79.254 GB.\n", + "39.311 GB of memory reserved.\n" + ] + } + ], + "source": [ + "# @title Show current memory stats\n", + "gpu_stats = torch.cuda.get_device_properties(0)\n", + "start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n", + "print(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\n", + "print(f\"{start_gpu_memory} GB of memory reserved.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r3WJRxG-zoeS" + }, + "source": [ + "We fixed a major gradient accumulation bug in all trainers. See [blog](https://unsloth.ai/blog/gradient) for more details." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "yqxqAZ7KJ4oL", + "outputId": "f401195d-1ff9-4a9f-8a8e-ccda6bf3c0ec" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", + " \\\\ /| Num examples = 10,000 | Num Epochs = 1 | Total steps = 30\n", + "O^O/ \\_/ \\ Batch size per device = 1 | Gradient accumulation steps = 4\n", + "\\ / Data Parallel GPUs = 1 | Total batch size (1 x 4 x 1) = 4\n", + " \"-____-\" Trainable parameters = 210,534,400 of 72,916,738,048 (0.29% trained)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Will smartly offload gradients to save VRAM!\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [30/30 09:43, Epoch 0/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
11.556900
21.601600
31.493700
41.538400
51.406000
61.220800
71.155600
81.113700
91.064300
101.042700
111.052500
120.971400
131.036500
141.033200
151.030700
160.990600
170.984900
180.959900
191.013500
201.064700
210.911000
220.931300
230.924600
241.008000
250.970500
260.935600
270.917000
280.846900
290.939000
300.901200

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "trainer_stats = trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pCqnaKmlO1U9", + "outputId": "23a21d88-0604-495f-eb05-92c34ce64765" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "608.5086 seconds used for training.\n", + "10.14 minutes used for training.\n", + "Peak reserved memory = 45.48 GB.\n", + "Peak reserved memory for training = 6.169 GB.\n", + "Peak reserved memory % of max memory = 57.385 %.\n", + "Peak reserved memory for training % of max memory = 7.784 %.\n" + ] + } + ], + "source": [ + "# @title Show final memory and time stats\n", + "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n", + "used_percentage = round(used_memory / max_memory * 100, 3)\n", + "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n", + "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n", + "print(\n", + " f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n", + ")\n", + "print(f\"Peak reserved memory = {used_memory} GB.\")\n", + "print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n", + "print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n", + "print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ekOmTR1hSNcr" + }, + "source": [ + "\n", + "### Inference\n", + "Let's run the model! You can change the instruction and input - leave the output blank!\n", + "\n", + "\n", + "\n", + "We use `min_p = 0.1` and `temperature = 1.5`. Read this [Tweet](https://x.com/menhguin/status/1826132708508213629) for more information on why." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'conversation': [{'content': 'You are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.',\n", + " 'role': 'system'},\n", + " {'content': 'I\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\nElasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n\\n\\nCan you help me debug this systematically and explain your reasoning?',\n", + " 'role': 'user'},\n", + " {'content': '', 'role': 'assistant'}],\n", + " 'metadata': {'base_commit': '160cfef6e1118061fa66d333e8c2a572f5d0a815',\n", + " 'dataset': 'princeton-nlp/SWE-bench',\n", + " 'generation_timestamp': '2025-08-19T23:35:38.773997',\n", + " 'has_patch': True,\n", + " 'instance_id': 'DataDog__integrations-core-10093',\n", + " 'original_index': 0,\n", + " 'reasoning_length': 0,\n", + " 'repo': 'DataDog/integrations-core',\n", + " 'source': 'swe-bench'},\n", + " 'original_example': {'instance_id': 'DataDog__integrations-core-10093',\n", + " 'patch': \"diff --git a/elastic/datadog_checks/elastic/elastic.py b/elastic/datadog_checks/elastic/elastic.py\\n--- a/elastic/datadog_checks/elastic/elastic.py\\n+++ b/elastic/datadog_checks/elastic/elastic.py\\n@@ -139,11 +139,17 @@ def _get_es_version(self):\\n try:\\n data = self._get_data(self._config.url, send_sc=False)\\n raw_version = data['version']['number']\\n+\\n self.set_metadata('version', raw_version)\\n # pre-release versions of elasticearch are suffixed with -rcX etc..\\n # peel that off so that the map below doesn't error out\\n raw_version = raw_version.split('-')[0]\\n version = [int(p) for p in raw_version.split('.')[0:3]]\\n+ if data['version'].get('distribution', '') == 'opensearch':\\n+ # Opensearch API is backwards compatible with ES 7.10.0\\n+ # https://opensearch.org/faq\\n+ self.log.debug('OpenSearch version %s detected', version)\\n+ version = [7, 10, 0]\\n except AuthenticationError:\\n raise\\n except Exception as e:\\n\",\n", + " 'problem_statement': 'Elasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n',\n", + " 'repo': 'DataDog/integrations-core'},\n", + " 'text': '<|im_start|>system\\nYou are an expert software debugging assistant. Help debug issues by analyzing problems systematically and providing clear, step-by-step reasoning.<|im_end|>\\n<|im_start|>user\\nI\\'m having an issue with the DataDog/integrations-core codebase. Here\\'s the problem:\\n\\nElasticsearch check fails when used with Opensearch\\n**Note:** If you have a feature request, you should [contact support](https://docs.datadoghq.com/help/) so the request can be properly tracked.\\r\\n\\r\\n**Output of the [info page](https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information)**\\r\\n\\r\\n```text\\r\\n elastic (1.24.0)\\r\\n ----------------\\r\\n Instance ID: elastic:e6126c6cf9b8eebe [ERROR]\\r\\n Configuration Source: file:/etc/datadog-agent/conf.d/elastic.yaml\\r\\n Total Runs: 121\\r\\n Metric Samples: Last Run: 0, Total: 0\\r\\n Events: Last Run: 0, Total: 0\\r\\n Service Checks: Last Run: 1, Total: 121\\r\\n Average Execution Time : 13ms\\r\\n Last Execution Date : 2021-09-10 06:00:16 UTC (1631253616000)\\r\\n Last Successful Execution Date : Never\\r\\n metadata:\\r\\n version.major: 1\\r\\n version.minor: 0\\r\\n version.patch: 0\\r\\n version.raw: 1.0.0\\r\\n version.scheme: semver\\r\\n Error: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n Traceback (most recent call last):\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\", line 897, in run\\r\\n self.check(instance)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 79, in check\\r\\n stats_data = self._get_data(stats_url)\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\", line 232, in _get_data\\r\\n resp.raise_for_status()\\r\\n File \"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\", line 940, in raise_for_status\\r\\n raise HTTPError(http_error_msg, response=self)\\r\\n requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\r\\n\\r\\n```\\r\\n\\r\\n**Additional environment details (Operating System, Cloud provider, etc):**\\r\\nFedora 33 running agent inside systemd-nspawn container. Container is running Opensearch 1.0.0 and datadog agent has been configured to monitor it.\\r\\n\\r\\n**Steps to reproduce the issue:**\\r\\n1. Install and run Opensearch\\r\\n2. Configure datadog agent Elasticsearch integration to monitor it\\r\\n3. Observe that the Elasticsearch check is failing\\r\\n\\r\\n**Describe the results you received:**\\r\\n\\r\\nThe Elasticsearch check fails in the following way:\\r\\n```text\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:261 in work) | check:elastic | Running check\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/check.go:81 in runCheck) | Running python check elastic elastic:878a21a7a9cbcd49\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET / HTTP/1.1\" 200 349\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:247) | request to url http://localhost:9200 returned: \\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | elastic:878a21a7a9cbcd49 | (elastic.py:147) | Elasticsearch version is [1, 0, 0]\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:226) | Starting new HTTP connection (1): localhost:9200\\r\\n2021-09-09 12:48:49 UTC | CORE | DEBUG | (pkg/collector/python/datadog_agent.go:126 in LogMessage) | - | (connectionpool.py:433) | http://localhost:9200 \"GET /_nodes/_local/stats?all=true HTTP/1.1\" 400 155\\r\\n2021-09-09 12:48:49 UTC | CORE | ERROR | (pkg/collector/runner/runner.go:292 in work) | Error running check elastic: [{\"message\": \"400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\", \"traceback\": \"Traceback (most recent call last):\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/base/checks/base.py\\\\\", line 897, in run\\\\n self.check(instance)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 79, in check\\\\n stats_data = self._get_data(stats_url)\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/datadog_checks/elastic/elastic.py\\\\\", line 232, in _get_data\\\\n resp.raise_for_status()\\\\n File \\\\\"/opt/datadog-agent/embedded/lib/python3.8/site-packages/requests/models.py\\\\\", line 940, in raise_for_status\\\\n raise HTTPError(http_error_msg, response=self)\\\\nrequests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:9200/_nodes/_local/stats?all=true\\\\n\"}]\\r\\n2021-09-09 12:48:49 UTC | CORE | INFO | (pkg/collector/runner/runner.go:327 in work) | check:elastic | Done running check\\r\\n```\\r\\n\\r\\n**Describe the results you expected:**\\r\\n\\r\\nOpensearch 1.0.0 is fully compatible with Elasticsearch 7.10.2 and datadog agent should be able to monitor it in the same way as the Elasticsearch.\\r\\n\\r\\n**Additional information you deem important (e.g. issue happens only occasionally):**\\r\\n\\r\\nThe issue seems to be that since Opensearch is reporting its version as `1.0.0` the Elasticsearch check interprets it as Elasticsearch `1.0.0` and for this reason adds `?all=true` to the URL when calling `/_nodes/_local/stats` API to collect the node stats. https://github.com/DataDog/integrations-core/blob/40d9ceb14c57c109e8f6371b1a4c677fa33e1669/elastic/datadog_checks/elastic/elastic.py#L215\\r\\n\\r\\nThe `?all=true` parameter has been depricated since Elasticsearch version `5.0.0` and results in HTTP 400 error when used with newer versions of Elasticsearch. All is working well when Elasticsearch check is monitoring Opensearch if `?all=true` has been left out when making requests to `/_nodes/_local/stats`.\\r\\n\\r\\nThe version information returned from Opensearch contains field called `distribution` which has value `opensearch` that could be used to detect that the check is actually running against an Opensearch server. Example here:\\r\\n```text\\r\\n{\\'name\\': \\'os-355e47a7-1\\', \\'cluster_name\\': \\'fc7a706a-9c51-40a1-9e1b-953d1546caa0\\', \\'cluster_uuid\\': \\'HrCNW5uhSZKtHxlJQz444g\\', \\'version\\': {\\'distribution\\': \\'opensearch\\', \\'number\\': \\'1.0.0\\', \\'build_type\\': \\'unknown\\', \\'build_hash\\': \\'unknown\\', \\'build_date\\': \\'2021-08-13T12:38:05.881264Z\\', \\'build_snapshot\\': False, \\'lucene_version\\': \\'8.8.2\\', \\'minimum_wire_compatibility_version\\': \\'6.8.0\\', \\'minimum_index_compatibility_version\\': \\'6.0.0-beta1\\'}, \\'tagline\\': \\'The OpenSearch Project: https://opensearch.org/\\'}\\r\\n```\\n\\n\\nCan you help me debug this systematically and explain your reasoning?<|im_end|>\\n<|im_start|>assistant\\n<|im_end|>\\n'}" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kR3gIAX-SM2q", + "outputId": "61a9d3ff-1ecd-4af2-a3c7-a17045af4fd0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n<|im_start|>user\\nHelp me resolve the code: see the error I was getting\\n\\n#!/usr/bin/env python3\\n\"\"\"\\nStyling Inference Script\\nProvides a command-line interface to run the styling inference pipeline\\n\"\"\"\\n\\nimport sys\\nimport os\\nimport subprocess\\nimport argparse\\nfrom pathlib import Path\\n\\ndef run_inference_with_config(config_path: str, instruction: str, input_text: str = \"\", max_tokens: int = 128, stream: bool = False):\\n \"\"\"Run inference using a YAML configuration file\"\"\"\\n print(f\"Running styling inference with config: {config_path}\")\\n print(f\"Instruction: {instruction}\")\\n print(f\"Input text: {input_text}\")\\n print(f\"Max tokens: {max_tokens}\")\\n print(f\"Streaming: {stream}\")\\n\\n cmd = [\\n \"python\", \"pipelines/styling/inference.py\",\\n \"--config\", config_path,\\n \"--instruction\", instruction,\\n \"--input-text\", input_text,\\n \"--max-tokens\", str(max_tokens)\\n ]\\n\\n if stream:\\n cmd.append(\"--stream\")\\n\\n print(f\"Running: {\\' \\'.join(cmd)}\")\\n\\n try:\\n result = subprocess.run(cmd, capture_output=True, text=True, check=True)\\n print(\"✅ Inference completed successfully!\")\\n print(\"Output:\")\\n print(result.stdout)\\n return result.stdout\\n except subprocess.CalledProcessError as e:\\n print(f\"❌ Inference failed: {e}\")\\n print(\"Error output:\")\\n print(e.stderr)\\n return None\\n\\ndef run_batch_inference_example(config_path: str, input_file: str, output_file: str, instruction: str, max_tokens: int = 128):\\n \"\"\"Run batch inference example\"\"\"\\n print(f\"=== Batch Inference Example ===\")\\n print(f\"Config: {config_path}\")\\n print(f\"Input file: {input_file}\")\\n print(f\"Output file: {output_file}\")\\n print(f\"Instruction: {instruction}\")\\n print(f\"Max tokens: {max_tokens}\")\\n\\n # Check if input file exists\\n if not Path(input_file).exists():\\n print(f\"❌ Input file not found: {input_file}\")\\n return False\\n\\n # Read input texts\\n with open(input_file, \\'r\\', encoding=\\'utf-8\\') as f:\\n input_texts = [line.strip() for line in f if line.strip()]\\n\\n print(f\"Found {len(input_texts)} texts to process\")\\n\\n # Process each text\\n results = []\\n for i, text in enumerate(input_texts):\\n print(f\"\\\\nProcessing text {i+1}/{len(input_texts)}: {text[:50]}...\")\\n result = run_inference_with_config(config_path, instruction, text, max_tokens)\\n if result:\\n results.append(result)\\n else:\\n print(f\"❌ Failed to process text {i+1}\")\\n\\n # Save results\\n with open(output_file, \\'w\\', encoding=\\'utf-8\\') as f:\\n for i, (input_text, result) in enumerate(zip(input_texts, results)):\\n f.write(f\"Input {i+1}: {input_text}\\\\n\")\\n f.write(f\"Output {i+1}: {result}\\\\n\")\\n f.write(\"-\" * 50 + \"\\\\n\")\\n\\n print(f\"✅ Batch inference completed! Results saved to: {output_file}\")\\n return True\\n\\ndef show_inference_features():\\n \"\"\"Show the features of the styling inference pipeline\"\"\"\\n print(\"=== Styling Inference Pipeline Features ===\")\\n print()\\n print(\"1. **Model Support**:\")\\n print(\" - Trained LoRA models from training pipeline\")\\n print(\" - Automatic model loading from config\")\\n print(\" - Native Unsloth inference optimization\")\\n print()\\n print(\"2. **Inference Modes**:\")\\n print(\" - Single text inference with instruction\")\\n print(\" - Batch file processing\")\\n print(\" - Streaming generation\")\\n print()\\n print(\"3. **Generation Control**:\")\\n print(\" - Configurable max tokens\")\\n print(\" - Same alpaca prompt format as training\")\\n print(\" - Automatic response extraction\")\\n print()\\n print(\"4. **Usage Examples**:\")\\n print(\" - Single inference: --instruction \\'style instruction\\' --input-text \\'your text\\'\")\\n print(\" - Streaming: add --stream flag\")\\n print(\" - Batch: use batch subcommand with input/output files\")\\n\\ndef create_inference_example():\\n \"\"\"Create an inference example using the formal style configuration\"\"\"\\n print(\"=== Inference Example: Formal Style Transfer ===\")\\n print()\\n\\n # Check if we have the required files\\n config_path = \"configs/styling/formal.yaml\"\\n\\n if not Path(config_path).exists():\\n print(f\"❌ Configuration file not found: {config_path}\")\\n print(\" Please run the data processor first to create the configuration\")\\n return False\\n\\n print(\"✅ Found configuration file!\")\\n print(f\" Config: {config_path}\")\\n print()\\n\\n # Example text\\n example_text = \"Hey, what\\'s up? I\\'m gonna go grab some food later.\"\\n\\n print(f\"Example text: {example_text}\")\\n print()\\n\\n # Run inference\\n success = run_inference_with_config(\\n config_path=config_path,\\n instruction=\"Rewrite the following text in a formal style\",\\n input_text=example_text\\n )\\n\\n if success:\\n print(\"✅ Example inference completed successfully!\")\\n return True\\n else:\\n print(\"❌ Example inference failed!\")\\n return False\\n\\ndef main():\\n \"\"\"Main inference function\"\"\"\\n parser = argparse.ArgumentParser(description=\"Styling Inference Pipeline\")\\n subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\\n\\n # Inference command\\n infer_parser = subparsers.add_parser(\"infer\", help=\"Run single inference\")\\n infer_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\\n infer_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\\n infer_parser.add_argument(\"--input-text\", type=str, default=\"\", help=\"Input text to style\")\\n infer_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\\n infer_parser.add_argument(\"--stream\", action=\"store_true\", help=\"Enable streaming generation\")\\n\\n # Batch inference command\\n batch_parser = subparsers.add_parser(\"batch\", help=\"Run batch inference\")\\n batch_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\\n batch_parser.add_argument(\"--input-file\", type=str, required=True, help=\"Input file with texts to style\")\\n batch_parser.add_argument(\"--output-file\", type=str, required=True, help=\"Output file for styled texts\")\\n batch_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\\n batch_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\\n\\n # Features command\\n subparsers.add_parser(\"features\", help=\"Show available features\")\\n\\n # Example command\\n subparsers.add_parser(\"example\", help=\"Run example inference\")\\n\\n args = parser.parse_args()\\n\\n if args.command == \"infer\":\\n run_inference_with_config(\\n args.config, \\n args.instruction, \\n args.input_text, \\n args.max_tokens, \\n args.stream\\n )\\n elif args.command == \"batch\":\\n run_batch_inference_example(args.config, args.input_file, args.output_file, args.instruction, args.max_tokens)\\n elif args.command == \"features\":\\n show_inference_features()\\n elif args.command == \"example\":\\n create_inference_example()\\n else:\\n parser.print_help()\\n\\nif __name__ == \"__main__\":\\n main()\\n\\nERROR: \\n(base) macbook@DynamoAi fine-tune-task % python scripts/styling/inference.py\\nusage: inference.py [-h] {infer,batch,features,example} ...\\n\\nStyling Inference Pipeline\\n\\npositional arguments:\\n {infer,batch,features,example}\\n Available commands\\n infer Run single inference\\n batch Run batch inference\\n features Show available features\\n example Run example inference\\n\\noptions:\\n -h, --help show this help message and exit\\n(base) macbook@DynamoAi fine-tune-task %<|im_end|>\\n<|im_start|>assistant\\nThe error you encountered is due to the absence of a command argument when running the script `inference.py`. Your script is designed to be used in conjunction with various subcommands such as `infer`, `batch`, `features`, or `example`. \\n\\nHere\\'s how these subcommands work:\\n\\n1. `infer`:']\n" + ] + } + ], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "\n", + "# Apply chat template to tokenizer\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template=\"qwen-2.5\",\n", + ")\n", + "\n", + "# Enable faster inference\n", + "FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n", + "\n", + "# Define the messages with properly formatted content\n", + "messages = [\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"\"\"Help me resolve the code: see the error I was getting\n", + "\n", + "#!/usr/bin/env python3\n", + "\\\"\\\"\\\"\n", + "Styling Inference Script\n", + "Provides a command-line interface to run the styling inference pipeline\n", + "\\\"\\\"\\\"\n", + "\n", + "import sys\n", + "import os\n", + "import subprocess\n", + "import argparse\n", + "from pathlib import Path\n", + "\n", + "def run_inference_with_config(config_path: str, instruction: str, input_text: str = \"\", max_tokens: int = 128, stream: bool = False):\n", + " \\\"\\\"\\\"Run inference using a YAML configuration file\\\"\\\"\\\"\n", + " print(f\"Running styling inference with config: {config_path}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Input text: {input_text}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " print(f\"Streaming: {stream}\")\n", + " \n", + " cmd = [\n", + " \"python\", \"pipelines/styling/inference.py\",\n", + " \"--config\", config_path,\n", + " \"--instruction\", instruction,\n", + " \"--input-text\", input_text,\n", + " \"--max-tokens\", str(max_tokens)\n", + " ]\n", + " \n", + " if stream:\n", + " cmd.append(\"--stream\")\n", + " \n", + " print(f\"Running: {' '.join(cmd)}\")\n", + " \n", + " try:\n", + " result = subprocess.run(cmd, capture_output=True, text=True, check=True)\n", + " print(\"✅ Inference completed successfully!\")\n", + " print(\"Output:\")\n", + " print(result.stdout)\n", + " return result.stdout\n", + " except subprocess.CalledProcessError as e:\n", + " print(f\"❌ Inference failed: {e}\")\n", + " print(\"Error output:\")\n", + " print(e.stderr)\n", + " return None\n", + "\n", + "def run_batch_inference_example(config_path: str, input_file: str, output_file: str, instruction: str, max_tokens: int = 128):\n", + " \\\"\\\"\\\"Run batch inference example\\\"\\\"\\\"\n", + " print(f\"=== Batch Inference Example ===\")\n", + " print(f\"Config: {config_path}\")\n", + " print(f\"Input file: {input_file}\")\n", + " print(f\"Output file: {output_file}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " \n", + " # Check if input file exists\n", + " if not Path(input_file).exists():\n", + " print(f\"❌ Input file not found: {input_file}\")\n", + " return False\n", + " \n", + " # Read input texts\n", + " with open(input_file, 'r', encoding='utf-8') as f:\n", + " input_texts = [line.strip() for line in f if line.strip()]\n", + " \n", + " print(f\"Found {len(input_texts)} texts to process\")\n", + " \n", + " # Process each text\n", + " results = []\n", + " for i, text in enumerate(input_texts):\n", + " print(f\"\\\\nProcessing text {i+1}/{len(input_texts)}: {text[:50]}...\")\n", + " result = run_inference_with_config(config_path, instruction, text, max_tokens)\n", + " if result:\n", + " results.append(result)\n", + " else:\n", + " print(f\"❌ Failed to process text {i+1}\")\n", + " \n", + " # Save results\n", + " with open(output_file, 'w', encoding='utf-8') as f:\n", + " for i, (input_text, result) in enumerate(zip(input_texts, results)):\n", + " f.write(f\"Input {i+1}: {input_text}\\\\n\")\n", + " f.write(f\"Output {i+1}: {result}\\\\n\")\n", + " f.write(\"-\" * 50 + \"\\\\n\")\n", + " \n", + " print(f\"✅ Batch inference completed! Results saved to: {output_file}\")\n", + " return True\n", + "\n", + "def show_inference_features():\n", + " \\\"\\\"\\\"Show the features of the styling inference pipeline\\\"\\\"\\\"\n", + " print(\"=== Styling Inference Pipeline Features ===\")\n", + " print()\n", + " print(\"1. **Model Support**:\")\n", + " print(\" - Trained LoRA models from training pipeline\")\n", + " print(\" - Automatic model loading from config\")\n", + " print(\" - Native Unsloth inference optimization\")\n", + " print()\n", + " print(\"2. **Inference Modes**:\")\n", + " print(\" - Single text inference with instruction\")\n", + " print(\" - Batch file processing\")\n", + " print(\" - Streaming generation\")\n", + " print()\n", + " print(\"3. **Generation Control**:\")\n", + " print(\" - Configurable max tokens\")\n", + " print(\" - Same alpaca prompt format as training\")\n", + " print(\" - Automatic response extraction\")\n", + " print()\n", + " print(\"4. **Usage Examples**:\")\n", + " print(\" - Single inference: --instruction 'style instruction' --input-text 'your text'\")\n", + " print(\" - Streaming: add --stream flag\")\n", + " print(\" - Batch: use batch subcommand with input/output files\")\n", + "\n", + "def create_inference_example():\n", + " \\\"\\\"\\\"Create an inference example using the formal style configuration\\\"\\\"\\\"\n", + " print(\"=== Inference Example: Formal Style Transfer ===\")\n", + " print()\n", + " \n", + " # Check if we have the required files\n", + " config_path = \"configs/styling/formal.yaml\"\n", + " \n", + " if not Path(config_path).exists():\n", + " print(f\"❌ Configuration file not found: {config_path}\")\n", + " print(\" Please run the data processor first to create the configuration\")\n", + " return False\n", + " \n", + " print(\"✅ Found configuration file!\")\n", + " print(f\" Config: {config_path}\")\n", + " print()\n", + " \n", + " # Example text\n", + " example_text = \"Hey, what's up? I'm gonna go grab some food later.\"\n", + " \n", + " print(f\"Example text: {example_text}\")\n", + " print()\n", + " \n", + " # Run inference\n", + " success = run_inference_with_config(\n", + " config_path=config_path,\n", + " instruction=\"Rewrite the following text in a formal style\",\n", + " input_text=example_text\n", + " )\n", + " \n", + " if success:\n", + " print(\"✅ Example inference completed successfully!\")\n", + " return True\n", + " else:\n", + " print(\"❌ Example inference failed!\")\n", + " return False\n", + "\n", + "def main():\n", + " \\\"\\\"\\\"Main inference function\\\"\\\"\\\"\n", + " parser = argparse.ArgumentParser(description=\"Styling Inference Pipeline\")\n", + " subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\n", + " \n", + " # Inference command\n", + " infer_parser = subparsers.add_parser(\"infer\", help=\"Run single inference\")\n", + " infer_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " infer_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " infer_parser.add_argument(\"--input-text\", type=str, default=\"\", help=\"Input text to style\")\n", + " infer_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " infer_parser.add_argument(\"--stream\", action=\"store_true\", help=\"Enable streaming generation\")\n", + " \n", + " # Batch inference command\n", + " batch_parser = subparsers.add_parser(\"batch\", help=\"Run batch inference\")\n", + " batch_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " batch_parser.add_argument(\"--input-file\", type=str, required=True, help=\"Input file with texts to style\")\n", + " batch_parser.add_argument(\"--output-file\", type=str, required=True, help=\"Output file for styled texts\")\n", + " batch_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " batch_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " \n", + " # Features command\n", + " subparsers.add_parser(\"features\", help=\"Show available features\")\n", + " \n", + " # Example command\n", + " subparsers.add_parser(\"example\", help=\"Run example inference\")\n", + " \n", + " args = parser.parse_args()\n", + " \n", + " if args.command == \"infer\":\n", + " run_inference_with_config(\n", + " args.config, \n", + " args.instruction, \n", + " args.input_text, \n", + " args.max_tokens, \n", + " args.stream\n", + " )\n", + " elif args.command == \"batch\":\n", + " run_batch_inference_example(args.config, args.input_file, args.output_file, args.instruction, args.max_tokens)\n", + " elif args.command == \"features\":\n", + " show_inference_features()\n", + " elif args.command == \"example\":\n", + " create_inference_example()\n", + " else:\n", + " parser.print_help()\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n", + "\n", + "ERROR: \n", + "(base) macbook@DynamoAi fine-tune-task % python scripts/styling/inference.py\n", + "usage: inference.py [-h] {infer,batch,features,example} ...\n", + "\n", + "Styling Inference Pipeline\n", + "\n", + "positional arguments:\n", + " {infer,batch,features,example}\n", + " Available commands\n", + " infer Run single inference\n", + " batch Run batch inference\n", + " features Show available features\n", + " example Run example inference\n", + "\n", + "options:\n", + " -h, --help show this help message and exit\n", + "(base) macbook@DynamoAi fine-tune-task %\"\"\"\n", + " },\n", + "]\n", + "\n", + "# Apply chat template and tokenize\n", + "inputs = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=True,\n", + " add_generation_prompt=True, # Must add for generation\n", + " return_tensors=\"pt\",\n", + ").to(\"cuda\")\n", + "\n", + "# Generate response\n", + "outputs = model.generate(\n", + " input_ids=inputs, \n", + " max_new_tokens=64, \n", + " use_cache=True,\n", + " temperature=1.5, \n", + " min_p=0.1\n", + ")\n", + "\n", + "# Decode the outputs\n", + "result = tokenizer.batch_decode(outputs)\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CrSvZObor0lY" + }, + "source": [ + " You can also use a `TextStreamer` for continuous inference - so you can see the generation token by token, instead of waiting the whole time!" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "e2pEuRb1r2Vg", + "outputId": "fe6590c1-dbd8-475b-c99b-17f88b302f9d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The error you're encountering is due to the fact that the script expects specific subcommands to be provided when running it. The error message indicates that no positional argument (subcommand) was given, which is required based on how the `argparse` module is set up in your script.\n", + "\n", + "### Explanation of the Error:\n", + "\n", + "When you run the script without any subcommands, the `args.command` will be `None`, leading to the `else` block where the help message is printed. This is why you see the usage information instead of the actual execution of the script.\n", + "\n", + "### Solution:\n", + "\n", + "To resolve this issue, you need to provide one of the expected subcommands when running the script. Here are the valid subcommands:\n", + "\n", + "1. **infer**: For running a single inference.\n", + "2. **batch**: For running batch inference.\n", + "3. **features**: To show the available features.\n", + "4. **example**: To run an example inference.\n", + "\n", + "### How to Run the Script Correctly:\n", + "\n", + "#### 1. Running Single Inference:\n", + "```bash\n", + "python scripts/styling/inference.py infer --config path/to/config.yaml --instruction \"Your instruction\" --input-text \"Your input text\"\n", + "```\n", + "\n", + "#### 2. Running Batch Inference:\n", + "```bash\n", + "python scripts/styling/inference.py batch --config path/to/config.yaml --input-file path/to/input.txt --output-file path/to/output.txt --instruction \"Your instruction\"\n", + "```\n", + "\n", + "#### 3. Showing Features:\n", + "```bash\n", + "python scripts/styling/inference.py features\n", + "```\n", + "\n", + "#### 4. Running Example Inference:\n", + "```bash\n", + "python scripts/styling/inference.py example\n", + "```\n", + "\n", + "### Additional Notes:\n", + "\n", + "- Ensure that the paths provided for `--config`, `--input-file`, and `--output-file` are correct and accessible.\n", + "- If you encounter any issues with file paths or configurations, check the existence of the files and the correctness of the YAML configuration.\n", + "\n", + "By providing the appropriate subcommand, you should be able to execute the script without encountering the error.<|im_end|>\n" + ] + } + ], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "\n", + "# Apply chat template to tokenizer\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template=\"qwen-2.5\",\n", + ")\n", + "\n", + "# Enable faster inference\n", + "FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n", + "\n", + "# Define the messages with properly formatted content\n", + "messages = [\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"\"\"Help me resolve the code: see the error I was getting\n", + "\n", + "#!/usr/bin/env python3\n", + "\\\"\\\"\\\"\n", + "Styling Inference Script\n", + "Provides a command-line interface to run the styling inference pipeline\n", + "\\\"\\\"\\\"\n", + "\n", + "import sys\n", + "import os\n", + "import subprocess\n", + "import argparse\n", + "from pathlib import Path\n", + "\n", + "def run_inference_with_config(config_path: str, instruction: str, input_text: str = \"\", max_tokens: int = 128, stream: bool = False):\n", + " \\\"\\\"\\\"Run inference using a YAML configuration file\\\"\\\"\\\"\n", + " print(f\"Running styling inference with config: {config_path}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Input text: {input_text}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " print(f\"Streaming: {stream}\")\n", + " \n", + " cmd = [\n", + " \"python\", \"pipelines/styling/inference.py\",\n", + " \"--config\", config_path,\n", + " \"--instruction\", instruction,\n", + " \"--input-text\", input_text,\n", + " \"--max-tokens\", str(max_tokens)\n", + " ]\n", + " \n", + " if stream:\n", + " cmd.append(\"--stream\")\n", + " \n", + " print(f\"Running: {' '.join(cmd)}\")\n", + " \n", + " try:\n", + " result = subprocess.run(cmd, capture_output=True, text=True, check=True)\n", + " print(\"✅ Inference completed successfully!\")\n", + " print(\"Output:\")\n", + " print(result.stdout)\n", + " return result.stdout\n", + " except subprocess.CalledProcessError as e:\n", + " print(f\"❌ Inference failed: {e}\")\n", + " print(\"Error output:\")\n", + " print(e.stderr)\n", + " return None\n", + "\n", + "def run_batch_inference_example(config_path: str, input_file: str, output_file: str, instruction: str, max_tokens: int = 128):\n", + " \\\"\\\"\\\"Run batch inference example\\\"\\\"\\\"\n", + " print(f\"=== Batch Inference Example ===\")\n", + " print(f\"Config: {config_path}\")\n", + " print(f\"Input file: {input_file}\")\n", + " print(f\"Output file: {output_file}\")\n", + " print(f\"Instruction: {instruction}\")\n", + " print(f\"Max tokens: {max_tokens}\")\n", + " \n", + " # Check if input file exists\n", + " if not Path(input_file).exists():\n", + " print(f\"❌ Input file not found: {input_file}\")\n", + " return False\n", + " \n", + " # Read input texts\n", + " with open(input_file, 'r', encoding='utf-8') as f:\n", + " input_texts = [line.strip() for line in f if line.strip()]\n", + " \n", + " print(f\"Found {len(input_texts)} texts to process\")\n", + " \n", + " # Process each text\n", + " results = []\n", + " for i, text in enumerate(input_texts):\n", + " print(f\"\\\\nProcessing text {i+1}/{len(input_texts)}: {text[:50]}...\")\n", + " result = run_inference_with_config(config_path, instruction, text, max_tokens)\n", + " if result:\n", + " results.append(result)\n", + " else:\n", + " print(f\"❌ Failed to process text {i+1}\")\n", + " \n", + " # Save results\n", + " with open(output_file, 'w', encoding='utf-8') as f:\n", + " for i, (input_text, result) in enumerate(zip(input_texts, results)):\n", + " f.write(f\"Input {i+1}: {input_text}\\\\n\")\n", + " f.write(f\"Output {i+1}: {result}\\\\n\")\n", + " f.write(\"-\" * 50 + \"\\\\n\")\n", + " \n", + " print(f\"✅ Batch inference completed! Results saved to: {output_file}\")\n", + " return True\n", + "\n", + "def show_inference_features():\n", + " \\\"\\\"\\\"Show the features of the styling inference pipeline\\\"\\\"\\\"\n", + " print(\"=== Styling Inference Pipeline Features ===\")\n", + " print()\n", + " print(\"1. **Model Support**:\")\n", + " print(\" - Trained LoRA models from training pipeline\")\n", + " print(\" - Automatic model loading from config\")\n", + " print(\" - Native Unsloth inference optimization\")\n", + " print()\n", + " print(\"2. **Inference Modes**:\")\n", + " print(\" - Single text inference with instruction\")\n", + " print(\" - Batch file processing\")\n", + " print(\" - Streaming generation\")\n", + " print()\n", + " print(\"3. **Generation Control**:\")\n", + " print(\" - Configurable max tokens\")\n", + " print(\" - Same alpaca prompt format as training\")\n", + " print(\" - Automatic response extraction\")\n", + " print()\n", + " print(\"4. **Usage Examples**:\")\n", + " print(\" - Single inference: --instruction 'style instruction' --input-text 'your text'\")\n", + " print(\" - Streaming: add --stream flag\")\n", + " print(\" - Batch: use batch subcommand with input/output files\")\n", + "\n", + "def create_inference_example():\n", + " \\\"\\\"\\\"Create an inference example using the formal style configuration\\\"\\\"\\\"\n", + " print(\"=== Inference Example: Formal Style Transfer ===\")\n", + " print()\n", + " \n", + " # Check if we have the required files\n", + " config_path = \"configs/styling/formal.yaml\"\n", + " \n", + " if not Path(config_path).exists():\n", + " print(f\"❌ Configuration file not found: {config_path}\")\n", + " print(\" Please run the data processor first to create the configuration\")\n", + " return False\n", + " \n", + " print(\"✅ Found configuration file!\")\n", + " print(f\" Config: {config_path}\")\n", + " print()\n", + " \n", + " # Example text\n", + " example_text = \"Hey, what's up? I'm gonna go grab some food later.\"\n", + " \n", + " print(f\"Example text: {example_text}\")\n", + " print()\n", + " \n", + " # Run inference\n", + " success = run_inference_with_config(\n", + " config_path=config_path,\n", + " instruction=\"Rewrite the following text in a formal style\",\n", + " input_text=example_text\n", + " )\n", + " \n", + " if success:\n", + " print(\"✅ Example inference completed successfully!\")\n", + " return True\n", + " else:\n", + " print(\"❌ Example inference failed!\")\n", + " return False\n", + "\n", + "def main():\n", + " \\\"\\\"\\\"Main inference function\\\"\\\"\\\"\n", + " parser = argparse.ArgumentParser(description=\"Styling Inference Pipeline\")\n", + " subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\n", + " \n", + " # Inference command\n", + " infer_parser = subparsers.add_parser(\"infer\", help=\"Run single inference\")\n", + " infer_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " infer_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " infer_parser.add_argument(\"--input-text\", type=str, default=\"\", help=\"Input text to style\")\n", + " infer_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " infer_parser.add_argument(\"--stream\", action=\"store_true\", help=\"Enable streaming generation\")\n", + " \n", + " # Batch inference command\n", + " batch_parser = subparsers.add_parser(\"batch\", help=\"Run batch inference\")\n", + " batch_parser.add_argument(\"--config\", type=str, required=True, help=\"Path to YAML configuration file\")\n", + " batch_parser.add_argument(\"--input-file\", type=str, required=True, help=\"Input file with texts to style\")\n", + " batch_parser.add_argument(\"--output-file\", type=str, required=True, help=\"Output file for styled texts\")\n", + " batch_parser.add_argument(\"--instruction\", type=str, required=True, help=\"Style instruction\")\n", + " batch_parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Maximum new tokens to generate\")\n", + " \n", + " # Features command\n", + " subparsers.add_parser(\"features\", help=\"Show available features\")\n", + " \n", + " # Example command\n", + " subparsers.add_parser(\"example\", help=\"Run example inference\")\n", + " \n", + " args = parser.parse_args()\n", + " \n", + " if args.command == \"infer\":\n", + " run_inference_with_config(\n", + " args.config, \n", + " args.instruction, \n", + " args.input_text, \n", + " args.max_tokens, \n", + " args.stream\n", + " )\n", + " elif args.command == \"batch\":\n", + " run_batch_inference_example(args.config, args.input_file, args.output_file, args.instruction, args.max_tokens)\n", + " elif args.command == \"features\":\n", + " show_inference_features()\n", + " elif args.command == \"example\":\n", + " create_inference_example()\n", + " else:\n", + " parser.print_help()\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n", + "\n", + "ERROR: \n", + "(base) macbook@DynamoAi fine-tune-task % python scripts/styling/inference.py\n", + "usage: inference.py [-h] {infer,batch,features,example} ...\n", + "\n", + "Styling Inference Pipeline\n", + "\n", + "positional arguments:\n", + " {infer,batch,features,example}\n", + " Available commands\n", + " infer Run single inference\n", + " batch Run batch inference\n", + " features Show available features\n", + " example Run example inference\n", + "\n", + "options:\n", + " -h, --help show this help message and exit\n", + "(base) macbook@DynamoAi fine-tune-task %\"\"\"\n", + " },\n", + "]\n", + "\n", + "# Apply chat template and tokenize\n", + "inputs = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=True,\n", + " add_generation_prompt=True, # Must add for generation\n", + " return_tensors=\"pt\",\n", + ").to(\"cuda\")\n", + "\n", + "# Decode the outputs\n", + "result = tokenizer.batch_decode(outputs)\n", + "\n", + "from transformers import TextStreamer\n", + "text_streamer = TextStreamer(tokenizer, skip_prompt = True)\n", + "_ = model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 1024,\n", + " use_cache = True, temperature = 0.2, min_p = 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uMuVrWbjAzhc" + }, + "source": [ + "\n", + "### Saving, loading finetuned models\n", + "To save the final model as LoRA adapters, either use Huggingface's `push_to_hub` for an online save or `save_pretrained` for a local save.\n", + "\n", + "**[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down!" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "upcOlWe7A1vc", + "outputId": "04d11b2b-7efd-4598-a60a-1873cfd3c431" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e60608a0567042ae9eef2a56b1760f96", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "README.md: 0%| | 0.00/601 [00:00 \u001b[39m\u001b[32m23\u001b[39m _ = \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstreamer\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtext_streamer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_new_tokens\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m128\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 24\u001b[39m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m1.5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmin_p\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m0.1\u001b[39;49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/peft/peft_model.py:1973\u001b[39m, in \u001b[36mPeftModelForCausalLM.generate\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1971\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m._enable_peft_forward_hooks(*args, **kwargs):\n\u001b[32m 1972\u001b[39m kwargs = {k: v \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m kwargs.items() \u001b[38;5;28;01mif\u001b[39;00m k \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.special_peft_forward_args}\n\u001b[32m-> \u001b[39m\u001b[32m1973\u001b[39m outputs = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mbase_model\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1974\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1975\u001b[39m outputs = \u001b[38;5;28mself\u001b[39m.base_model.generate(**kwargs)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:1794\u001b[39m, in \u001b[36munsloth_fast_generate\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1792\u001b[39m \u001b[38;5;66;03m# Mixed precision autocast\u001b[39;00m\n\u001b[32m 1793\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m torch.inference_mode(), torch.autocast(device_type = DEVICE_TYPE, dtype = dtype):\n\u001b[32m-> \u001b[39m\u001b[32m1794\u001b[39m output = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_old_generate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1795\u001b[39m \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[32m 1797\u001b[39m \u001b[38;5;66;03m# Return accelerate back\u001b[39;00m\n\u001b[32m 1798\u001b[39m \u001b[38;5;66;03m# if accelerate_new_send_to_device is not None:\u001b[39;00m\n\u001b[32m 1799\u001b[39m \u001b[38;5;66;03m# accelerate.utils.operations.send_to_device = accelerate_old_send_to_device\u001b[39;00m\n\u001b[32m 1800\u001b[39m \u001b[38;5;66;03m# pass\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/torch/utils/_contextlib.py:120\u001b[39m, in \u001b[36mcontext_decorator..decorate_context\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 117\u001b[39m \u001b[38;5;129m@functools\u001b[39m.wraps(func)\n\u001b[32m 118\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mdecorate_context\u001b[39m(*args, **kwargs):\n\u001b[32m 119\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[32m--> \u001b[39m\u001b[32m120\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/transformers/generation/utils.py:2617\u001b[39m, in \u001b[36mGenerationMixin.generate\u001b[39m\u001b[34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, use_model_defaults, custom_generate, **kwargs)\u001b[39m\n\u001b[32m 2609\u001b[39m input_ids, model_kwargs = \u001b[38;5;28mself\u001b[39m._expand_inputs_for_generation(\n\u001b[32m 2610\u001b[39m input_ids=input_ids,\n\u001b[32m 2611\u001b[39m expand_size=generation_config.num_return_sequences,\n\u001b[32m 2612\u001b[39m is_encoder_decoder=\u001b[38;5;28mself\u001b[39m.config.is_encoder_decoder,\n\u001b[32m 2613\u001b[39m **model_kwargs,\n\u001b[32m 2614\u001b[39m )\n\u001b[32m 2616\u001b[39m \u001b[38;5;66;03m# 12. run sample (it degenerates to greedy search when `generation_config.do_sample=False`)\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m2617\u001b[39m result = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sample\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 2618\u001b[39m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2619\u001b[39m \u001b[43m \u001b[49m\u001b[43mlogits_processor\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprepared_logits_processor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2620\u001b[39m \u001b[43m \u001b[49m\u001b[43mstopping_criteria\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprepared_stopping_criteria\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2621\u001b[39m \u001b[43m \u001b[49m\u001b[43mgeneration_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgeneration_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2622\u001b[39m \u001b[43m \u001b[49m\u001b[43msynced_gpus\u001b[49m\u001b[43m=\u001b[49m\u001b[43msynced_gpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2623\u001b[39m \u001b[43m \u001b[49m\u001b[43mstreamer\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstreamer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2624\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2625\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2627\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m generation_mode \u001b[38;5;129;01min\u001b[39;00m (GenerationMode.BEAM_SAMPLE, GenerationMode.BEAM_SEARCH):\n\u001b[32m 2628\u001b[39m \u001b[38;5;66;03m# 11. interleave input_ids with `num_beams` additional sequences per batch\u001b[39;00m\n\u001b[32m 2629\u001b[39m input_ids, model_kwargs = \u001b[38;5;28mself\u001b[39m._expand_inputs_for_generation(\n\u001b[32m 2630\u001b[39m input_ids=input_ids,\n\u001b[32m 2631\u001b[39m expand_size=generation_config.num_beams,\n\u001b[32m 2632\u001b[39m is_encoder_decoder=\u001b[38;5;28mself\u001b[39m.config.is_encoder_decoder,\n\u001b[32m 2633\u001b[39m **model_kwargs,\n\u001b[32m 2634\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/transformers/generation/utils.py:3601\u001b[39m, in \u001b[36mGenerationMixin._sample\u001b[39m\u001b[34m(self, input_ids, logits_processor, stopping_criteria, generation_config, synced_gpus, streamer, **model_kwargs)\u001b[39m\n\u001b[32m 3599\u001b[39m is_prefill = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m 3600\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m3601\u001b[39m outputs = \u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m 3603\u001b[39m \u001b[38;5;66;03m# synced_gpus: don't waste resources running the code we don't need; kwargs must be updated before skipping\u001b[39;00m\n\u001b[32m 3604\u001b[39m model_kwargs = \u001b[38;5;28mself\u001b[39m._update_model_kwargs_for_generation(\n\u001b[32m 3605\u001b[39m outputs,\n\u001b[32m 3606\u001b[39m model_kwargs,\n\u001b[32m 3607\u001b[39m is_encoder_decoder=\u001b[38;5;28mself\u001b[39m.config.is_encoder_decoder,\n\u001b[32m 3608\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py:1773\u001b[39m, in \u001b[36mModule._wrapped_call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1771\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._compiled_call_impl(*args, **kwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 1772\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1773\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py:1784\u001b[39m, in \u001b[36mModule._call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1779\u001b[39m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[32m 1780\u001b[39m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[32m 1781\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_pre_hooks\n\u001b[32m 1782\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[32m 1783\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[32m-> \u001b[39m\u001b[32m1784\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1786\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1787\u001b[39m called_always_called_hooks = \u001b[38;5;28mset\u001b[39m()\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:1166\u001b[39m, in \u001b[36mCausalLM_fast_forward.._CausalLM_fast_forward\u001b[39m\u001b[34m(self, input_ids, causal_mask, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict, num_logits_to_keep, logits_to_keep, *args, **kwargs)\u001b[39m\n\u001b[32m 1148\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_CausalLM_fast_forward\u001b[39m(\n\u001b[32m 1149\u001b[39m \u001b[38;5;28mself\u001b[39m,\n\u001b[32m 1150\u001b[39m input_ids: torch.LongTensor = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m (...)\u001b[39m\u001b[32m 1163\u001b[39m *args, **kwargs,\n\u001b[32m 1164\u001b[39m ) -> Union[Tuple, CausalLMOutputWithPast]:\n\u001b[32m 1165\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m past_key_values \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1166\u001b[39m outputs = \u001b[43mfast_forward_inference\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1167\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 1168\u001b[39m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1169\u001b[39m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1170\u001b[39m \u001b[43m \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1171\u001b[39m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1172\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1173\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1174\u001b[39m causal_mask = xformers.attn_bias.LowerTriangularMask() \u001b[38;5;28;01mif\u001b[39;00m HAS_XFORMERS \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:1117\u001b[39m, in \u001b[36m_LlamaModel_fast_forward_inference..LlamaModel_fast_forward_inference_custom\u001b[39m\u001b[34m(self, input_ids, past_key_values, position_ids, attention_mask)\u001b[39m\n\u001b[32m 1109\u001b[39m residual.copy_(X) \u001b[38;5;66;03m# residual = X\u001b[39;00m\n\u001b[32m 1110\u001b[39m X = fast_rms_layernorm_inference(\n\u001b[32m 1111\u001b[39m decoder_layer.post_attention_layernorm,\n\u001b[32m 1112\u001b[39m X,\n\u001b[32m (...)\u001b[39m\u001b[32m 1115\u001b[39m variance = variance,\n\u001b[32m 1116\u001b[39m )\n\u001b[32m-> \u001b[39m\u001b[32m1117\u001b[39m X = \u001b[43mmlp_fast_forward_inference\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1118\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecoder_layer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmlp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1119\u001b[39m \u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1120\u001b[39m \u001b[43m \u001b[49m\u001b[43mtemp_gate\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_gates\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_index\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1121\u001b[39m \u001b[43m \u001b[49m\u001b[43mtemp_up\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_ups\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_index\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1122\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1123\u001b[39m X += residual\n\u001b[32m 1125\u001b[39m next_decoder_cache.append(present_key_value)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/models/llama.py:411\u001b[39m, in \u001b[36mfast_swiglu_inference\u001b[39m\u001b[34m(self, X, temp_gate, temp_up, gate_multiplier, down_multiplier)\u001b[39m\n\u001b[32m 408\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m gate_multiplier \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 409\u001b[39m gate *= gate_multiplier\n\u001b[32m--> \u001b[39m\u001b[32m411\u001b[39m up = \u001b[43mfast_linear_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m \u001b[49m\u001b[43mup_proj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_up\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 413\u001b[39m gate = torch_nn_functional_silu(gate, inplace = \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 414\u001b[39m gate *= up\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/dist-packages/unsloth/kernels/utils.py:680\u001b[39m, in \u001b[36mfast_linear_forward\u001b[39m\u001b[34m(proj, X, temp_lora, out)\u001b[39m\n\u001b[32m 678\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m bsz == \u001b[32m1\u001b[39m:\n\u001b[32m 679\u001b[39m out = out.view(out_dim)\n\u001b[32m--> \u001b[39m\u001b[32m680\u001b[39m temp_lora = \u001b[43mtorch_mv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlora_A\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_fast_lora\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m.\u001b[49m\u001b[43mravel\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_lora\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 681\u001b[39m out.addmv_(lora_B._fast_lora, temp_lora, alpha = lora_S)\n\u001b[32m 682\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "\u001b[31mKeyboardInterrupt\u001b[39m: " + ] + } + ], + "source": [ + "if False:\n", + " from unsloth import FastLanguageModel\n", + " model, tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name = \"qwen_2_5_72b_debug_research\", # YOUR MODEL YOU USED FOR TRAINING\n", + " max_seq_length = max_seq_length,\n", + " dtype = dtype,\n", + " load_in_4bit = load_in_4bit,\n", + " )\n", + " FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n", + "\n", + "messages = [\n", + " {\"role\": \"user\", \"content\": \"Describe a tall tower in the capital of France.\"},\n", + "]\n", + "inputs = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize = True,\n", + " add_generation_prompt = True, # Must add for generation\n", + " return_tensors = \"pt\",\n", + ").to(\"cuda\")\n", + "\n", + "from transformers import TextStreamer\n", + "text_streamer = TextStreamer(tokenizer, skip_prompt = True)\n", + "_ = model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 128,\n", + " use_cache = True, temperature = 1.5, min_p = 0.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QQMjaNrjsU5_" + }, + "source": [ + "You can also use Hugging Face's `AutoModelForPeftCausalLM`. Only use this if you do not have `unsloth` installed. It can be hopelessly slow, since `4bit` model downloading is not supported, and Unsloth's **inference is 2x faster**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yFfaXG0WsQuE" + }, + "outputs": [], + "source": [ + "if False:\n", + " # I highly do NOT suggest - use Unsloth if possible\n", + " from peft import AutoPeftModelForCausalLM\n", + " from transformers import AutoTokenizer\n", + "\n", + " model = AutoPeftModelForCausalLM.from_pretrained(\n", + " \"lora_model\", # YOUR MODEL YOU USED FOR TRAINING\n", + " load_in_4bit=load_in_4bit,\n", + " )\n", + " tokenizer = AutoTokenizer.from_pretrained(\"lora_model\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f422JgM9sdVT" + }, + "source": [ + "### Saving to float16 for VLLM\n", + "\n", + "We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "iHjt_SMYsd3P" + }, + "outputs": [], + "source": [ + "# Merge to 16bit\n", + "if False: model.save_pretrained_merged(\"model\", tokenizer, save_method = \"merged_16bit\",)\n", + "if False: model.push_to_hub_merged(\"hf/model\", tokenizer, save_method = \"merged_16bit\", token = \"\")\n", + "\n", + "# Merge to 4bit\n", + "if False: model.save_pretrained_merged(\"model\", tokenizer, save_method = \"merged_4bit\",)\n", + "if False: model.push_to_hub_merged(\"hf/model\", tokenizer, save_method = \"merged_4bit\", token = \"\")\n", + "\n", + "# Just LoRA adapters\n", + "if False:\n", + " model.save_pretrained(\"model\")\n", + " tokenizer.save_pretrained(\"model\")\n", + "if False:\n", + " model.push_to_hub(\"hf/model\", token = \"\")\n", + " tokenizer.push_to_hub(\"hf/model\", token = \"\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TCv4vXHd61i7" + }, + "source": [ + "### GGUF / llama.cpp Conversion\n", + "To save to `GGUF` / `llama.cpp`, we support it natively now! We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF.\n", + "\n", + "Some supported quant methods (full list on our [Wiki page](https://github.com/unslothai/unsloth/wiki#gguf-quantization-options)):\n", + "* `q8_0` - Fast conversion. High resource use, but generally acceptable.\n", + "* `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K.\n", + "* `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K.\n", + "\n", + "[**NEW**] To finetune and auto export to Ollama, try our [Ollama notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FqfebeAdT073" + }, + "outputs": [], + "source": [ + "# Save to 8bit Q8_0\n", + "if False: model.save_pretrained_gguf(\"model\", tokenizer,)\n", + "# Remember to go to https://huggingface.co/settings/tokens for a token!\n", + "# And change hf to your username!\n", + "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, token = \"\")\n", + "\n", + "# Save to 16bit GGUF\n", + "if False: model.save_pretrained_gguf(\"model\", tokenizer, quantization_method = \"f16\")\n", + "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, quantization_method = \"f16\", token = \"\")\n", + "\n", + "# Save to q4_k_m GGUF\n", + "if False: model.save_pretrained_gguf(\"model\", tokenizer, quantization_method = \"q4_k_m\")\n", + "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, quantization_method = \"q4_k_m\", token = \"\")\n", + "\n", + "# Save to multiple GGUF options - much faster if you want multiple!\n", + "if False:\n", + " model.push_to_hub_gguf(\n", + " \"hf/model\", # Change hf to your username!\n", + " tokenizer,\n", + " quantization_method = [\"q4_k_m\", \"q8_0\", \"q5_k_m\",],\n", + " token = \"\", # Get a token at https://huggingface.co/settings/tokens\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kBvrJqNr6L9g" + }, + "source": [ + "Now, use the `model-unsloth.gguf` file or `model-unsloth-Q4_K_M.gguf` file in llama.cpp.\n", + "\n", + "And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n", + "\n", + "Some other links:\n", + "1. Train your own reasoning model - Llama GRPO notebook [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb)\n", + "2. Saving finetunes to Ollama. [Free notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb)\n", + "3. Llama 3.2 Vision finetuning - Radiography use case. [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb)\n", + "6. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://docs.unsloth.ai/get-started/unsloth-notebooks)!\n", + "\n", + "

\n", + " \n", + " \n", + " \n", + "\n", + " Join Discord if you need help + ⭐️ Star us on Github ⭐️\n", + "
\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "019f19eb060545539da7628564caaf5e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "02d6ff36400543d196671dcb609a3554": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "035a4fa66ce8447fbbed4adfd986726b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "03ea102e7b20453b939c9e514f7a61d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "04c78130763c4dfc93a86c788a6e01e8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0526bd2f6b014475bc4279fb6ad44ab0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1a89be1d70a043a7ab316fe731a4af95", + "max": 7512, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_cfd8bbe5291845f6b886415ece13f7f7", + "value": 7512 + } + }, + "052dc013a23944ca8d1de2d0c7ab8141": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "06ad2446cdb64c4db134046da4cfd0a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "071495bad0e143ce8b5514d75ce5d435": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9dd47b04490548bbb03c204544cc022e", + "placeholder": "​", + "style": "IPY_MODEL_524a6d91d0924912b4ce4a2f4c87e76a", + "value": " 7.03M/7.03M [00:00<00:00, 44.7MB/s]" + } + }, + "08d6a0314e864384aa3e163fb2224c35": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0dd1d1bffd654340b8d21af1f0eab11d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0df17c7f00c949c7b8444b91754fa4dd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0dfde06ab9d54e6b8c563d380f4b6026": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c4df1f0a6cb34bca9a84cb734ffcd6c7", + "placeholder": "​", + "style": "IPY_MODEL_8a87d368abe3434c82f99fb23e717e5e", + "value": " 613/613 [00:00<00:00, 36.5kB/s]" + } + }, + "0e66ab8fb2384f09a303a034fbde6247": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f65529309064ff781015f38375a34ce": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "101d6d98e8bb4a1ab216ddac084314ec": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1105390239d846898e653a7bfd092ee6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "167f32a4f26a4466a3c9b94fb7aa34b2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "16946595a9064832a233c889e01818c9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "16dee8a92a9e4fbd908eeda7193fc190": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "17f32b453e4841a383c210853a36da84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "18a1a798b3824b33b73f345520583d91": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1a89be1d70a043a7ab316fe731a4af95": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1ad8199be4dd49ea8a9638d3e53d3d74": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8409a21d6d594569aca7721796af52ba", + "max": 7031863, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c883d3a00bc542db88244a71b924e1f0", + "value": 7031863 + } + }, + "1c971ddb47764c6aa4f766c5abd8cd71": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1ccf810e39ea4fbcb81f465c3b6bd601": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "213d400d9c704b9aaed21ad5a91c91a7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d09a3f6c289c4608a8dfcf53157bf1b6", + "IPY_MODEL_1ad8199be4dd49ea8a9638d3e53d3d74", + "IPY_MODEL_071495bad0e143ce8b5514d75ce5d435" + ], + "layout": "IPY_MODEL_7d7b05e1f1f642acb8f5ba6afdf6d82f" + } + }, + "23cf64d2dff94432af9cf6a7f99744ae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "244e4cf841cd483b8e9e165c77c471f0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b47587f1ba944ad9850fcf555cdbcf12", + "placeholder": "​", + "style": "IPY_MODEL_82806a720dad4704ba224812d9018bc2", + "value": " 100000/100000 [00:02<00:00, 41148.75 examples/s]" + } + }, + "24f12af7e8664d72ae38b06a713294a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a7d2b0d1a685407fa0633eb56f3676db", + "placeholder": "​", + "style": "IPY_MODEL_7dbc6c270eff43569640b25e450601ab", + "value": " 117M/117M [00:00<00:00, 137MB/s]" + } + }, + "254584c067e146809ab71a7a85d7c7a4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "25ce0fb1aed24b0cb1419ac3761109f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9d15aa6b077044f4a7aba7c3d543c984", + "placeholder": "​", + "style": "IPY_MODEL_9a2c98e1c8d5491791e4d7a1390cf81f", + "value": "added_tokens.json: 100%" + } + }, + "25d1aaf77b304450877876f5c8bd38fb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "260322195cc9480aaeb1173315a93894": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_55f6a6b67a754ef89732a29531dcf34a", + "placeholder": "​", + "style": "IPY_MODEL_25d1aaf77b304450877876f5c8bd38fb", + "value": " 5.00G/5.00G [00:33<00:00, 181MB/s]" + } + }, + "27f38f1ed0a14f119ba6b8f48662f5b9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2853c5ef931d422aa5a7622bc2f61bb2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f15f91be9d1e47da8079d1d0d48f7842", + "placeholder": "​", + "style": "IPY_MODEL_17f32b453e4841a383c210853a36da84", + "value": "Loading checkpoint shards: 100%" + } + }, + "2983c2011c194d0b88383d3bbf356845": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "298be59fbb97424080819b7a16db2024": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b851f3f552ad41fe97e9d7b72ea71b1c", + "IPY_MODEL_963f33764e744624bd33a8252511c0f2", + "IPY_MODEL_5f9574f2a91742d6a11be231ae60b4a6" + ], + "layout": "IPY_MODEL_895dccba65a5482c94d4c74371412241" + } + }, + "2cd72f4bdc1b41a4a2894c553d335dc5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3032a337e20945198e3678f91c371bc9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e1d1a5a6b94741a4b22e254c504264c0", + "IPY_MODEL_63b67e250d8349f88ee276ef732168c9", + "IPY_MODEL_cae54a63540742c3bb7a0412a5c55add" + ], + "layout": "IPY_MODEL_ae9f5ce0eefc4a8a98c2599a0fbf890a" + } + }, + "31400dbf077840769de1cb7f63cec131": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3215fa6f5c0446b9b6b4a0d15dd33032": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bccfa4e6adfd40d3afff51f5ab038422", + "placeholder": "​", + "style": "IPY_MODEL_d6292ba7c64e4e3f9baf68905657ec56", + "value": " 100000/100000 [00:07<00:00, 16491.66 examples/s]" + } + }, + "32b193ccdb024d5993e42b4e0efd1631": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3371554f74674d2da66fc8c794822c24": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "38db9cffac4a42759c3c1be2e5a4dd3f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3973d4960a584bcebe080796c99104f1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "39c1e706d28a46218940e01190b2843a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3ae2a86a77a34f61a204bd1355dda8b3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3b74d36ab9974076b06ffe7b545d582f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3c58773e281144f39081dc3414f9f4c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_71b65997837d405ebc3e92cb8b498faa", + "max": 2776833, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_bf4e7a04363d477cb36586a0fbcfc64e", + "value": 2776833 + } + }, + "413290be48d2410c93e2da6848affa39": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "41c627c3ffb144eca3cd45f490e61a30": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d67e9d729bee4325a5a01a90aecb52d0", + "placeholder": "​", + "style": "IPY_MODEL_7756a42c820745389107246cb6a35072", + "value": "model-00001-of-00002.safetensors: 100%" + } + }, + "44fdce605bf54fba80f81524461259bf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4949d26418b944f0a7d4c8e69a49f3f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_edf75f353d9645ba8796e2652d636166", + "IPY_MODEL_b6c160bf3d3043f8840ab4b7431301ca", + "IPY_MODEL_cacb47001cd341abae219b526401616d" + ], + "layout": "IPY_MODEL_1c971ddb47764c6aa4f766c5abd8cd71" + } + }, + "4d78f66177ca43f6b1efa5588078c0e4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_553a899890fb42348e5447ed43d5fd9b", + "placeholder": "​", + "style": "IPY_MODEL_b985370c39e64ea0943e1edda65a6bed", + "value": "tokenizer_config.json: 100%" + } + }, + "4e38a0d4d6ca440388951805ce4b6104": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4f5037bad4c849f19ccfd7b0f57f6721": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d69b25177b214cfb877907376763b4e6", + "IPY_MODEL_de85541764644bd685f7a950115c2544", + "IPY_MODEL_5808b66051e943c7ae58cf2dfebf2b41" + ], + "layout": "IPY_MODEL_5ee0f800a0934bc596a2b8249709c6e9" + } + }, + "524a6d91d0924912b4ce4a2f4c87e76a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "52a3d29efbd04ec7858876606b31465c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5341de66236b459ca83a154fb65e0596": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0dd1d1bffd654340b8d21af1f0eab11d", + "max": 613, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_f16ef7e24cd64841a3e2d8988d286c58", + "value": 613 + } + }, + "553a899890fb42348e5447ed43d5fd9b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "55aec8e495624b2faa5492e612807dbb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2cd72f4bdc1b41a4a2894c553d335dc5", + "placeholder": "​", + "style": "IPY_MODEL_9e31bce1bd2c4cf48724f9f5d9c5282f", + "value": " 7.51k/7.51k [00:00<00:00, 420kB/s]" + } + }, + "55f6a6b67a754ef89732a29531dcf34a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "56146e26a4054d87a843ace9758b7100": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "57c9f679f77241d0b6f9bf794dad2b9b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "5808b66051e943c7ae58cf2dfebf2b41": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6f98b1e08f954e82adc4565a0b7556c1", + "placeholder": "​", + "style": "IPY_MODEL_a263b7c2a34e41dabc94d78b99835ad8", + "value": " 100000/100000 [00:52<00:00, 1874.99 examples/s]" + } + }, + "594a5e73c6d24e8a8f98400c2ee99636": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_38db9cffac4a42759c3c1be2e5a4dd3f", + "placeholder": "​", + "style": "IPY_MODEL_32b193ccdb024d5993e42b4e0efd1631", + "value": " 982/982 [00:00<00:00, 20.3kB/s]" + } + }, + "5998b125c6a64ef886fc51d745b5301d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e2a1b3b26e84490c99b305d4a651cda4", + "IPY_MODEL_5341de66236b459ca83a154fb65e0596", + "IPY_MODEL_0dfde06ab9d54e6b8c563d380f4b6026" + ], + "layout": "IPY_MODEL_27f38f1ed0a14f119ba6b8f48662f5b9" + } + }, + "5d053c12d1ae427fa34c553d4759e25d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5d97ec03ac9544409f51c2ae1759dc8b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5ee0f800a0934bc596a2b8249709c6e9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f63b5e75daf44ada6af8dd6922a58ce": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b2def88c44ee4b608faaf38148b30c18", + "placeholder": "​", + "style": "IPY_MODEL_08d6a0314e864384aa3e163fb2224c35", + "value": " 1.67M/1.67M [00:00<00:00, 34.6MB/s]" + } + }, + "5f9574f2a91742d6a11be231ae60b4a6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_035a4fa66ce8447fbbed4adfd986726b", + "placeholder": "​", + "style": "IPY_MODEL_16dee8a92a9e4fbd908eeda7193fc190", + "value": " 210k/210k [00:00<00:00, 3.13MB/s]" + } + }, + "6178ea883ce546fd9f433d5a683d2d18": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_99c57c528f2d4c6284022efba3b9e81d", + "max": 4998641583, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1ccf810e39ea4fbcb81f465c3b6bd601", + "value": 4998641107 + } + }, + "617973da3a3f4b1f86729365d12f9875": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "63b67e250d8349f88ee276ef732168c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f65529309064ff781015f38375a34ce", + "max": 4933695760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_7350d931506e4e99b55a03751c453515", + "value": 4933695290 + } + }, + "6765cb1a549f45e998ccb19139e2e8a3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c2d9520cf1134cc182c2612bfd912875", + "IPY_MODEL_a53fab9c72f6468aaffd5bab3765af49", + "IPY_MODEL_244e4cf841cd483b8e9e165c77c471f0" + ], + "layout": "IPY_MODEL_8bbeb59a93de469682466e54bc6c35a7" + } + }, + "6b4a5dee5e3143a9b69dede345b7187d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6b81f7175b224cb4a4320b33d969c146": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_41c627c3ffb144eca3cd45f490e61a30", + "IPY_MODEL_6178ea883ce546fd9f433d5a683d2d18", + "IPY_MODEL_260322195cc9480aaeb1173315a93894" + ], + "layout": "IPY_MODEL_c69c5e05efc8464f86c662b9ae931538" + } + }, + "6cbf4a502296486d9b0d8588dc123732": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6db5bed4b1c4478c97e07a7ab32d1845": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6e2018997fc545d8bb37c862fff0c751": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6f98b1e08f954e82adc4565a0b7556c1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "709b591f878747ee9a22c01bb8fa2c74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7135057d6cb84a77a04a8db3c6c3f336": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71b65997837d405ebc3e92cb8b498faa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "734f1dd7724d49f5a196552da6abf192": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b00ce2847b33492b9eba44d065ea7f06", + "placeholder": "​", + "style": "IPY_MODEL_7ba890d72f4245cb8f760f252ff80caa", + "value": "merges.txt: 100%" + } + }, + "7350d931506e4e99b55a03751c453515": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7382f0c83e7b44d3a19bbfda0d3723ee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "760c42f3b2054820acf6b1b6da11356a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "76a9c48d89e44cfc9d30cdf3b56a4f74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7756a42c820745389107246cb6a35072": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "789512128fa84d3481a086db62c44c37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e658cae26b3f457882ef5fe90309ed38", + "placeholder": "​", + "style": "IPY_MODEL_4e38a0d4d6ca440388951805ce4b6104", + "value": " 2/2 [00:53<00:00, 26.72s/it]" + } + }, + "7942f894ea8f4268b9c2b6ad83b2f279": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9c40f79c7efb4b758c6af85ba105fe4c", + "max": 2, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_413290be48d2410c93e2da6848affa39", + "value": 2 + } + }, + "7972d106d5414a218441942913aa3e3c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_167f32a4f26a4466a3c9b94fb7aa34b2", + "placeholder": "​", + "style": "IPY_MODEL_d5682db4b5cd4bb4abdb6ff7258caefa", + "value": " 2.78M/2.78M [00:00<00:00, 11.2MB/s]" + } + }, + "79ce9732a3af4920a64598318a491398": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7a285248d8fa4378860cd5e899613057": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ba5b4ef26bc14dadbd444c7a039a2995", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_18a1a798b3824b33b73f345520583d91", + "value": 100000 + } + }, + "7b2494ece7db465ca088670a7f8ffbef": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7ba890d72f4245cb8f760f252ff80caa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7cbc2991535b440b97e02a9df10b306a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7d7b05e1f1f642acb8f5ba6afdf6d82f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7dbc6c270eff43569640b25e450601ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "815aec1656224eacabc076618f7f067a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4d78f66177ca43f6b1efa5588078c0e4", + "IPY_MODEL_0526bd2f6b014475bc4279fb6ad44ab0", + "IPY_MODEL_55aec8e495624b2faa5492e612807dbb" + ], + "layout": "IPY_MODEL_e6eccabf31e545a2b06c7650c9781d05" + } + }, + "82806a720dad4704ba224812d9018bc2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8409a21d6d594569aca7721796af52ba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8892267f311b4cf0a5267ce193c5a646": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fa44d5de08014ac48013167025a3de1b", + "max": 2, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c9f72c0ebbee4aad8023b2082ce59f34", + "value": 2 + } + }, + "89520764d8ef4805805055bebd21273e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_56146e26a4054d87a843ace9758b7100", + "placeholder": "​", + "style": "IPY_MODEL_02d6ff36400543d196671dcb609a3554", + "value": " 632/632 [00:00<00:00, 45.3kB/s]" + } + }, + "895dccba65a5482c94d4c74371412241": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8a87d368abe3434c82f99fb23e717e5e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8b1c036ade754137a675d55d6ad12e2c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6cbf4a502296486d9b0d8588dc123732", + "placeholder": "​", + "style": "IPY_MODEL_7382f0c83e7b44d3a19bbfda0d3723ee", + "value": " 100000/100000 [04:03<00:00, 425.91 examples/s]" + } + }, + "8bbeb59a93de469682466e54bc6c35a7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8bcfc8c1bd6c474fabac2dfff5ca829d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_23cf64d2dff94432af9cf6a7f99744ae", + "placeholder": "​", + "style": "IPY_MODEL_bfef7ae67a7b43ed816246be0ed19a79", + "value": "Map: 100%" + } + }, + "8c6346ded74b4614871353d09b685ab3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7cbc2991535b440b97e02a9df10b306a", + "placeholder": "​", + "style": "IPY_MODEL_ac9c54ff279a4080b7485d669989db6d", + "value": "Map (num_proc=4): 100%" + } + }, + "8cf481de1931406e97951a2adfbdeee0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_101d6d98e8bb4a1ab216ddac084314ec", + "max": 116531415, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_617973da3a3f4b1f86729365d12f9875", + "value": 116531404 + } + }, + "8dd6e110704f40c995b9f0f6951cbb74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8f9648d8465c414ea4699a63fbf936a9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8fe160b637b24a74a4a97bdd9b1858d1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "93ccb0dfad4e4d60a7e7f42402bff80d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8c6346ded74b4614871353d09b685ab3", + "IPY_MODEL_7a285248d8fa4378860cd5e899613057", + "IPY_MODEL_8b1c036ade754137a675d55d6ad12e2c" + ], + "layout": "IPY_MODEL_b9304f02c4774b6eb161da6fa6b009ff" + } + }, + "963f33764e744624bd33a8252511c0f2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7135057d6cb84a77a04a8db3c6c3f336", + "max": 210129, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a12e1cf419bd446b9823fc924431cae4", + "value": 210129 + } + }, + "97359398bd724bf1acf5c4370bb7eaa2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "975b0754e56546d4bd7103b4f28a37c5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e38e2b0f246d4b369487fd405be38e15", + "placeholder": "​", + "style": "IPY_MODEL_6b4a5dee5e3143a9b69dede345b7187d", + "value": " 100000/100000 [00:27<00:00, 4638.49 examples/s]" + } + }, + "994b86b8732346de8b0789f366068549": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "99c57c528f2d4c6284022efba3b9e81d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9a2c98e1c8d5491791e4d7a1390cf81f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9b816b35709d4b4fbedd183e6a7aae5a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9c40f79c7efb4b758c6af85ba105fe4c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9d15aa6b077044f4a7aba7c3d543c984": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9dd47b04490548bbb03c204544cc022e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9e31bce1bd2c4cf48724f9f5d9c5282f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9e7f78f4d932445daecd09dcf7b37859": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_994b86b8732346de8b0789f366068549", + "placeholder": "​", + "style": "IPY_MODEL_3b74d36ab9974076b06ffe7b545d582f", + "value": "Downloading shards: 100%" + } + }, + "a12e1cf419bd446b9823fc924431cae4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a263b7c2a34e41dabc94d78b99835ad8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a2951310cc79437a8dac88cadd9ed1e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_254584c067e146809ab71a7a85d7c7a4", + "placeholder": "​", + "style": "IPY_MODEL_44fdce605bf54fba80f81524461259bf", + "value": "train-00000-of-00001.parquet: 100%" + } + }, + "a53fab9c72f6468aaffd5bab3765af49": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8fe160b637b24a74a4a97bdd9b1858d1", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_57c9f679f77241d0b6f9bf794dad2b9b", + "value": 100000 + } + }, + "a6baf9fd477342fe8474022ac8e7a6ed": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a7d2b0d1a685407fa0633eb56f3676db": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8f4a39bb9d74a9bb4138c8bed4186e5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ac9c54ff279a4080b7485d669989db6d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "acda49009d124138843b187c2aa9e410": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "adc2a364ac1b4901acbae39562402743": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae9f5ce0eefc4a8a98c2599a0fbf890a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b00ce2847b33492b9eba44d065ea7f06": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b1a3ff316c374f509898c21c8618dcc9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b27f191668cf4d1cbf9705fe51e3c3df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e2018997fc545d8bb37c862fff0c751", + "placeholder": "​", + "style": "IPY_MODEL_acda49009d124138843b187c2aa9e410", + "value": "README.md: 100%" + } + }, + "b2def88c44ee4b608faaf38148b30c18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b468a2d515f943979af3d1921cda5594": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e7cf601a9b6342cfb38376989e08c5a9", + "max": 632, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ca23b61a88bf4e609075619b9d1111ef", + "value": 632 + } + }, + "b47587f1ba944ad9850fcf555cdbcf12": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b608a884c04541ca88253a72d1d81cb6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_760c42f3b2054820acf6b1b6da11356a", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_cf74cf0282cd409498112783ef3068aa", + "value": 100000 + } + }, + "b6c160bf3d3043f8840ab4b7431301ca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8dd6e110704f40c995b9f0f6951cbb74", + "max": 266, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c2dd2c13d62d48e09a68373f10dcd95a", + "value": 266 + } + }, + "b718bba8164047ccbb046d3475fb0f12": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_25ce0fb1aed24b0cb1419ac3761109f7", + "IPY_MODEL_b468a2d515f943979af3d1921cda5594", + "IPY_MODEL_89520764d8ef4805805055bebd21273e" + ], + "layout": "IPY_MODEL_adc2a364ac1b4901acbae39562402743" + } + }, + "b83dcf4897f544678d85a072cfc8bcca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a6baf9fd477342fe8474022ac8e7a6ed", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3371554f74674d2da66fc8c794822c24", + "value": 100000 + } + }, + "b851f3f552ad41fe97e9d7b72ea71b1c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_39c1e706d28a46218940e01190b2843a", + "placeholder": "​", + "style": "IPY_MODEL_5d97ec03ac9544409f51c2ae1759dc8b", + "value": "model.safetensors.index.json: 100%" + } + }, + "b9304f02c4774b6eb161da6fa6b009ff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b985370c39e64ea0943e1edda65a6bed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ba0b2bd35ce44100be27bc749642e1cc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_16946595a9064832a233c889e01818c9", + "placeholder": "​", + "style": "IPY_MODEL_ebdb9c8af7be4743b2e3c1308bddc0a5", + "value": "Standardizing format: 100%" + } + }, + "ba5b4ef26bc14dadbd444c7a039a2995": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bbac5526b00745c982a175bd180cbd24": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bccfa4e6adfd40d3afff51f5ab038422": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bf4e7a04363d477cb36586a0fbcfc64e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "bfef7ae67a7b43ed816246be0ed19a79": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c186068d654b4ba3a97105956d52083e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c19fa4dea6524d37843cbf5786b26221": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fba4df061ec84f4abe521ee628c53b62", + "max": 982, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e0cf51730c35468ea3074be96df8b8a0", + "value": 982 + } + }, + "c2d9520cf1134cc182c2612bfd912875": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0df17c7f00c949c7b8444b91754fa4dd", + "placeholder": "​", + "style": "IPY_MODEL_a8f4a39bb9d74a9bb4138c8bed4186e5", + "value": "Generating train split: 100%" + } + }, + "c2dd2c13d62d48e09a68373f10dcd95a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c4df1f0a6cb34bca9a84cb734ffcd6c7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c69c5e05efc8464f86c662b9ae931538": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c883d3a00bc542db88244a71b924e1f0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c9c96e91dc6347e1b6566570e925e40f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2853c5ef931d422aa5a7622bc2f61bb2", + "IPY_MODEL_8892267f311b4cf0a5267ce193c5a646", + "IPY_MODEL_789512128fa84d3481a086db62c44c37" + ], + "layout": "IPY_MODEL_8f9648d8465c414ea4699a63fbf936a9" + } + }, + "c9f72c0ebbee4aad8023b2082ce59f34": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ca23b61a88bf4e609075619b9d1111ef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cacb47001cd341abae219b526401616d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_97359398bd724bf1acf5c4370bb7eaa2", + "placeholder": "​", + "style": "IPY_MODEL_06ad2446cdb64c4db134046da4cfd0a5", + "value": " 266/266 [00:00<00:00, 18.5kB/s]" + } + }, + "cae54a63540742c3bb7a0412a5c55add": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_04c78130763c4dfc93a86c788a6e01e8", + "placeholder": "​", + "style": "IPY_MODEL_e3a4437a1f8b4710a77983b82f0ebe30", + "value": " 4.93G/4.93G [01:06<00:00, 16.8MB/s]" + } + }, + "cf74cf0282cd409498112783ef3068aa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cfd8bbe5291845f6b886415ece13f7f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d09a3f6c289c4608a8dfcf53157bf1b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_019f19eb060545539da7628564caaf5e", + "placeholder": "​", + "style": "IPY_MODEL_3973d4960a584bcebe080796c99104f1", + "value": "tokenizer.json: 100%" + } + }, + "d5682db4b5cd4bb4abdb6ff7258caefa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d6292ba7c64e4e3f9baf68905657ec56": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d67e9d729bee4325a5a01a90aecb52d0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d69b25177b214cfb877907376763b4e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_052dc013a23944ca8d1de2d0c7ab8141", + "placeholder": "​", + "style": "IPY_MODEL_52a3d29efbd04ec7858876606b31465c", + "value": "Map: 100%" + } + }, + "d6d14db2e4894ad18b899cafb148b871": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d71c0ead75c2411587677493ad59256b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d94c60b5a75249a894c761c7baf12ff4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "da6b6fcb41f74c73b0b3460ba6d4c7dd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9e7f78f4d932445daecd09dcf7b37859", + "IPY_MODEL_7942f894ea8f4268b9c2b6ad83b2f279", + "IPY_MODEL_dfed5a49746e413bbaa44aa82a61228c" + ], + "layout": "IPY_MODEL_6db5bed4b1c4478c97e07a7ab32d1845" + } + }, + "de85541764644bd685f7a950115c2544": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d94c60b5a75249a894c761c7baf12ff4", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e37375168ae64bc1a8c8c1451dc2c4b2", + "value": 100000 + } + }, + "dee9866d465e43c99e61f71b819d9994": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba0b2bd35ce44100be27bc749642e1cc", + "IPY_MODEL_b83dcf4897f544678d85a072cfc8bcca", + "IPY_MODEL_3215fa6f5c0446b9b6b4a0d15dd33032" + ], + "layout": "IPY_MODEL_79ce9732a3af4920a64598318a491398" + } + }, + "dfed5a49746e413bbaa44aa82a61228c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1105390239d846898e653a7bfd092ee6", + "placeholder": "​", + "style": "IPY_MODEL_9b816b35709d4b4fbedd183e6a7aae5a", + "value": " 2/2 [01:40<00:00, 53.30s/it]" + } + }, + "e0cf51730c35468ea3074be96df8b8a0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e1d1a5a6b94741a4b22e254c504264c0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d71c0ead75c2411587677493ad59256b", + "placeholder": "​", + "style": "IPY_MODEL_c186068d654b4ba3a97105956d52083e", + "value": "model-00002-of-00002.safetensors: 100%" + } + }, + "e2a1b3b26e84490c99b305d4a651cda4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7b2494ece7db465ca088670a7f8ffbef", + "placeholder": "​", + "style": "IPY_MODEL_03ea102e7b20453b939c9e514f7a61d5", + "value": "special_tokens_map.json: 100%" + } + }, + "e37375168ae64bc1a8c8c1451dc2c4b2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e38e2b0f246d4b369487fd405be38e15": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e3a4437a1f8b4710a77983b82f0ebe30": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e658cae26b3f457882ef5fe90309ed38": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e6eccabf31e545a2b06c7650c9781d05": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e7371eff6f4c458792abd1e05e147110": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b27f191668cf4d1cbf9705fe51e3c3df", + "IPY_MODEL_c19fa4dea6524d37843cbf5786b26221", + "IPY_MODEL_594a5e73c6d24e8a8f98400c2ee99636" + ], + "layout": "IPY_MODEL_709b591f878747ee9a22c01bb8fa2c74" + } + }, + "e7cf601a9b6342cfb38376989e08c5a9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ebdb9c8af7be4743b2e3c1308bddc0a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ec4c9d4fe69645d78243987a8c12d503": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_734f1dd7724d49f5a196552da6abf192", + "IPY_MODEL_fcea53819cc441ec907174ee79ccc8ee", + "IPY_MODEL_5f63b5e75daf44ada6af8dd6922a58ce" + ], + "layout": "IPY_MODEL_31400dbf077840769de1cb7f63cec131" + } + }, + "edf75f353d9645ba8796e2652d636166": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_76a9c48d89e44cfc9d30cdf3b56a4f74", + "placeholder": "​", + "style": "IPY_MODEL_2983c2011c194d0b88383d3bbf356845", + "value": "generation_config.json: 100%" + } + }, + "f15f91be9d1e47da8079d1d0d48f7842": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f16ef7e24cd64841a3e2d8988d286c58": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "f4a36fec7d7a41e49bf125cc5cc98b0f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a2951310cc79437a8dac88cadd9ed1e6", + "IPY_MODEL_8cf481de1931406e97951a2adfbdeee0", + "IPY_MODEL_24f12af7e8664d72ae38b06a713294a5" + ], + "layout": "IPY_MODEL_0e66ab8fb2384f09a303a034fbde6247" + } + }, + "f6bee8f11d694040ae78e6f8616070d0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f7646ab3a3ff4106b33ab158c0296ae7", + "IPY_MODEL_3c58773e281144f39081dc3414f9f4c9", + "IPY_MODEL_7972d106d5414a218441942913aa3e3c" + ], + "layout": "IPY_MODEL_b1a3ff316c374f509898c21c8618dcc9" + } + }, + "f7646ab3a3ff4106b33ab158c0296ae7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bbac5526b00745c982a175bd180cbd24", + "placeholder": "​", + "style": "IPY_MODEL_5d053c12d1ae427fa34c553d4759e25d", + "value": "vocab.json: 100%" + } + }, + "f93b699993824be7a005d54d96cd6361": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "fa44d5de08014ac48013167025a3de1b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fba4df061ec84f4abe521ee628c53b62": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fcea53819cc441ec907174ee79ccc8ee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d6d14db2e4894ad18b899cafb148b871", + "max": 1671853, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_f93b699993824be7a005d54d96cd6361", + "value": 1671853 + } + }, + "ffcb4731dff04faa8323dceae528bce3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8bcfc8c1bd6c474fabac2dfff5ca829d", + "IPY_MODEL_b608a884c04541ca88253a72d1d81cb6", + "IPY_MODEL_975b0754e56546d4bd7103b4f28a37c5" + ], + "layout": "IPY_MODEL_3ae2a86a77a34f61a204bd1355dda8b3" + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/pipelines/instruct/.ipynb_checkpoints/data_processor-checkpoint.py b/pipelines/instruct/.ipynb_checkpoints/data_processor-checkpoint.py index e69de29..30a912c 100644 --- a/pipelines/instruct/.ipynb_checkpoints/data_processor-checkpoint.py +++ b/pipelines/instruct/.ipynb_checkpoints/data_processor-checkpoint.py @@ -0,0 +1,913 @@ +import json +import pandas as pd +import numpy as np +from pathlib import Path +from typing import Dict, List, Optional, Union, Any, Tuple +from datasets import Dataset, load_dataset +import os +from dataclasses import dataclass +from abc import ABC, abstractmethod +from sklearn.model_selection import train_test_split +import re +import argparse +import sys +import yaml + +@dataclass +class InstructConfig: + """Configuration for instruction fine-tuning tasks""" + # Data source configuration + data_source: str = "custom" # "huggingface" or "custom" + dataset_name: Optional[str] = None # For Hugging Face datasets + data_path: Optional[str] = None # For custom datasets + data_format: str = "jsonl" # jsonl, json + + # Field mapping - conversation data specific + conversation_field: str = "conversation" # Field containing conversation array + + # Data processing + max_samples: Optional[int] = None + train_split: float = 0.8 + validation_split: float = 0.1 + test_split: float = 0.1 + + # Text preprocessing + clean_text: bool = True + min_length: int = 10 + max_length: int = 2048 + + # Output configuration + output_format: str = "conversation" # conversation, alpaca + output_dir: str = "./data/processed/instruct" + + # Hugging Face specific + hf_split: str = "train" + hf_cache_dir: Optional[str] = None + + # Split configuration + test_split_from: str = "train" + val_split_from: str = "train" + + # Custom data specific + encoding: str = "utf-8" + +class ConversationValidator: + """Validates conversation data quality and format""" + + @staticmethod + def validate_conversation_data(data: Dict[str, List[Dict]], config: InstructConfig, is_processed: bool = False) -> Tuple[bool, List[str]]: + """Validate conversation dataset splits""" + errors = [] + + # Check if we have the expected splits + expected_splits = ["train", "validation", "test"] + for split in expected_splits: + if split not in data: + errors.append(f"Missing '{split}' split") + elif split == "train" and not data[split]: + errors.append(f"Train split cannot be empty") + + if errors: + return False, errors + + total_samples = sum(len(split_data) for split_data in data.values()) + print(f"Validating {total_samples} total samples across all splits...") + + # Determine field names based on whether data is processed or not + conversation_field = "conversation" if not is_processed else "conversation" + + # Validate each split + for split_name, split_data in data.items(): + if not split_data: + print(f"Skipping validation for empty {split_name} split") + continue + + print(f"Validating {split_name} split with {len(split_data)} samples...") + + # Check required fields + missing_conversation_count = 0 + + for i, item in enumerate(split_data): + if conversation_field not in item: + errors.append(f"Missing conversation field '{conversation_field}' in {split_name} split, item {i}") + missing_conversation_count += 1 + else: + # Validate conversation structure + conversation = item[conversation_field] + if not isinstance(conversation, list): + errors.append(f"Conversation field must be a list in {split_name} split, item {i}") + else: + # Validate each turn in conversation + for j, turn in enumerate(conversation): + if not isinstance(turn, dict): + errors.append(f"Each conversation turn must be a dict in {split_name} split, item {i}, turn {j}") + continue + + # Check for required fields in conversation turn + if "role" not in turn: + errors.append(f"Missing 'role' field in conversation turn {j}, {split_name} split, item {i}") + if "content" not in turn: + errors.append(f"Missing 'content' field in conversation turn {j}, {split_name} split, item {i}") + + # Validate role values + if "role" in turn and turn["role"] not in ["user", "assistant", "system"]: + errors.append(f"Invalid role '{turn['role']}' in conversation turn {j}, {split_name} split, item {i}. Must be 'user', 'assistant', or 'system'") + + print(f"{split_name} - Items missing conversation field: {missing_conversation_count}") + + # Show sample of processed data for debugging + if split_data: + print(f"Sample conversation from {split_name}:") + for i in range(min(2, len(split_data))): + item = split_data[i] + conversation = item.get(conversation_field, []) + print(f" Item {i} conversation length: {len(conversation)} turns") + for j, turn in enumerate(conversation[:3]): # Show first 3 turns + role = turn.get("role", "unknown") + content = turn.get("content", "")[:100] + "..." if len(turn.get("content", "")) > 100 else turn.get("content", "") + print(f" Turn {j}: {role} -> '{content}'") + + return len(errors) == 0, errors + + @staticmethod + def analyze_conversation_dataset(data: Dict[str, List[Dict]], config: InstructConfig, is_processed: bool = False) -> Dict[str, Any]: + """Analyze conversation dataset characteristics across all splits""" + analysis = { + "splits": {}, + "overall": { + "total_samples": 0, + "split_sizes": {}, + "conversation_stats": { + "total_turns": 0, + "avg_turns_per_conversation": 0, + "role_distribution": {"user": 0, "assistant": 0, "system": 0} + } + } + } + + conversation_field = "conversation" if not is_processed else "conversation" + total_turns = 0 + total_conversations = 0 + role_counts = {"user": 0, "assistant": 0, "system": 0} + + # Analyze each split + for split_name, split_data in data.items(): + if not split_data: + split_analysis = { + "total_samples": 0, + "conversation_stats": {}, + "missing_values": {} + } + analysis["splits"][split_name] = split_analysis + analysis["overall"]["split_sizes"][split_name] = 0 + continue + + split_analysis = { + "total_samples": len(split_data), + "conversation_stats": {}, + "missing_values": {} + } + + # Conversation statistics + split_turns = 0 + split_conversations = len(split_data) + split_role_counts = {"user": 0, "assistant": 0, "system": 0} + conversation_lengths = [] + + for item in split_data: + conversation = item.get(conversation_field, []) + if isinstance(conversation, list): + conversation_lengths.append(len(conversation)) + split_turns += len(conversation) + for turn in conversation: + if isinstance(turn, dict) and "role" in turn: + role = turn["role"] + if role in split_role_counts: + split_role_counts[role] += 1 + + if conversation_lengths: + split_analysis["conversation_stats"] = { + "total_turns": split_turns, + "avg_turns_per_conversation": np.mean(conversation_lengths), + "min_turns": min(conversation_lengths), + "max_turns": max(conversation_lengths), + "median_turns": np.median(conversation_lengths), + "role_distribution": split_role_counts + } + + # Missing values + missing_count = sum(1 for item in split_data if not item.get(conversation_field)) + split_analysis["missing_values"][conversation_field] = missing_count + + analysis["splits"][split_name] = split_analysis + analysis["overall"]["total_samples"] += len(split_data) + analysis["overall"]["split_sizes"][split_name] = len(split_data) + + # Accumulate overall stats + total_turns += split_turns + total_conversations += split_conversations + for role, count in split_role_counts.items(): + role_counts[role] += count + + # Calculate overall conversation stats + if total_conversations > 0: + analysis["overall"]["conversation_stats"]["total_turns"] = total_turns + analysis["overall"]["conversation_stats"]["avg_turns_per_conversation"] = total_turns / total_conversations + analysis["overall"]["conversation_stats"]["role_distribution"] = role_counts + + return analysis + +class BaseInstructDataLoader(ABC): + """Abstract base class for instruction data loaders""" + + @abstractmethod + def load(self, config: InstructConfig) -> Dict[str, List[Dict]]: + """Load data and return dictionary with train/val/test splits""" + pass + + @abstractmethod + def preprocess(self, data: Dict[str, List[Dict]], config: InstructConfig) -> Dict[str, List[Dict]]: + """Apply preprocessing steps to all splits""" + pass + +class HuggingFaceInstructDataLoader(BaseInstructDataLoader): + """Load conversation datasets from Hugging Face Hub""" + + def load(self, config: InstructConfig) -> Dict[str, List[Dict]]: + """Load dataset from Hugging Face Hub with flexible split handling""" + if not config.dataset_name: + raise ValueError("Dataset name is required for Hugging Face datasets") + + print(f"Loading Hugging Face conversation dataset: {config.dataset_name}") + + try: + dataset = load_dataset( + config.dataset_name, + cache_dir=config.hf_cache_dir + ) + + available_splits = list(dataset.keys()) + print(f"Available splits in dataset: {available_splits}") + + splits_data = { + "train": [], + "validation": [], + "test": [] + } + + # Handle train split + if "train" in available_splits: + train_dataset = dataset["train"] + print(f"Using 'train' split with {len(train_dataset)} samples") + splits_data["train"] = list(train_dataset) + else: + print("No 'train' split found in dataset!") + raise ValueError(f"Dataset {config.dataset_name} does not have a 'train' split") + + # Handle validation and test splits (similar logic to styling pipeline) + # ... [validation and test split handling logic similar to styling pipeline] + + # Apply max_samples limit if specified + if config.max_samples: + for split_name in splits_data: + if splits_data[split_name]: + original_size = len(splits_data[split_name]) + splits_data[split_name] = splits_data[split_name][:config.max_samples] + print(f"Limited {split_name} split from {original_size} to {len(splits_data[split_name])} samples") + + print(f"Successfully loaded dataset {config.dataset_name}") + return splits_data + + except Exception as e: + print(f"Error loading dataset {config.dataset_name}: {e}") + raise + + def preprocess(self, data: Dict[str, List[Dict]], config: InstructConfig) -> Dict[str, List[Dict]]: + """Apply preprocessing steps to all splits separately""" + processed_splits = {} + + print(f"=== PREPROCESSING CONVERSATION DATA ===") + + for split_name, split_data in data.items(): + print(f"Processing {split_name} split with {len(split_data)} items...") + + processed_data = [] + processed_count = 0 + skipped_count = 0 + + for i, item in enumerate(split_data): + processed_item = self._preprocess_item(item, config) + if processed_item is not None: + processed_data.append(processed_item) + processed_count += 1 + else: + skipped_count += 1 + + processed_splits[split_name] = processed_data + print(f"{split_name} - Preprocessed {processed_count} samples, skipped {skipped_count} samples") + + return processed_splits + + def _preprocess_item(self, item: Dict, config: InstructConfig) -> Optional[Dict]: + """Preprocess a single conversation item""" + conversation = item.get(config.conversation_field, []) + + if not isinstance(conversation, list) or not conversation: + return None + + # Validate conversation structure + valid_conversation = [] + for turn in conversation: + if not isinstance(turn, dict): + continue + if "role" not in turn or "content" not in turn: + continue + if turn["role"] not in ["user", "assistant", "system"]: + continue + + content = str(turn["content"]).strip() + if len(content) < config.min_length or len(content) > config.max_length: + continue + + if config.clean_text: + content = self._clean_text(content) + + valid_conversation.append({ + "role": turn["role"], + "content": content + }) + + if len(valid_conversation) < 2: # Need at least 2 turns for a conversation + return None + + return {"conversation": valid_conversation} + + def _clean_text(self, text: str) -> str: + """Clean and normalize text""" + if not isinstance(text, str): + return "" + + # Remove extra whitespace + text = re.sub(r'\s+', ' ', text).strip() + return text + +class CustomInstructDataLoader(BaseInstructDataLoader): + """Load custom conversation datasets from local files""" + + def load(self, config: InstructConfig) -> Dict[str, List[Dict]]: + """Load custom conversation dataset from local file and create splits""" + if not config.data_path: + raise ValueError("Data path is required for custom datasets") + + file_path = Path(config.data_path) + + if not file_path.exists(): + raise FileNotFoundError(f"Data file not found: {file_path}") + + print(f"Loading custom conversation dataset: {file_path}") + + if config.data_format == "jsonl": + raw_data = self._load_jsonl(file_path, config) + elif config.data_format == "json": + raw_data = self._load_json(file_path, config) + else: + raise ValueError(f"Unsupported format: {config.data_format}") + + if config.max_samples: + raw_data = raw_data[:config.max_samples] + + print(f"Loaded {len(raw_data)} conversation samples from {file_path}") + + # Create splits from the raw data + splits_data = self._create_splits(raw_data, config) + + return splits_data + + def _create_splits(self, data: List[Dict], config: InstructConfig) -> Dict[str, List[Dict]]: + """Create train/validation/test splits from raw data""" + print(f"Creating splits from {len(data)} conversation samples...") + + # Handle very small datasets + if len(data) < 3: + print(f"Dataset has only {len(data)} samples. Using all data for training.") + return { + "train": data, + "validation": [], + "test": [] + } + + # Calculate split sizes + total_samples = len(data) + + # Adjust split ratios if dataset is too small + if total_samples < 10: + config.train_split = 0.6 + config.validation_split = 0.2 + config.test_split = 0.2 + print(f"Small dataset detected. Adjusted split ratios to: train={config.train_split}, val={config.validation_split}, test={config.test_split}") + + val_size = max(1, int(total_samples * config.validation_split)) + test_size = max(1, int(total_samples * config.test_split)) + train_size = total_samples - val_size - test_size + + # Ensure train split has at least 1 sample + if train_size < 1: + if val_size > 1: + val_size -= 1 + train_size += 1 + elif test_size > 1: + test_size -= 1 + train_size += 1 + + print(f"Split sizes: train={train_size}, validation={val_size}, test={test_size}") + + # Create splits + if val_size == 0 and test_size == 0: + splits_data = { + "train": data, + "validation": [], + "test": [] + } + elif val_size == 0: + train_data, test_data = train_test_split(data, test_size=test_size, random_state=42) + splits_data = { + "train": train_data, + "validation": [], + "test": test_data + } + elif test_size == 0: + train_data, val_data = train_test_split(data, test_size=val_size, random_state=42) + splits_data = { + "train": train_data, + "validation": val_data, + "test": [] + } + else: + # Full three-way split + train_data, temp_data = train_test_split( + data, + test_size=val_size + test_size, + random_state=42 + ) + + val_data, test_data = train_test_split( + temp_data, + test_size=test_size, + random_state=42 + ) + + splits_data = { + "train": train_data, + "validation": val_data, + "test": test_data + } + + print(f"Created conversation splits:") + print(f" Train: {len(splits_data['train'])} samples") + print(f" Validation: {len(splits_data['validation'])} samples") + print(f" Test: {len(splits_data['test'])} samples") + + return splits_data + + def _load_jsonl(self, file_path: Path, config: InstructConfig) -> List[Dict]: + """Load JSONL file""" + data = [] + with open(file_path, 'r', encoding=config.encoding) as f: + for line_num, line in enumerate(f, 1): + if line.strip(): + try: + data.append(json.loads(line)) + except json.JSONDecodeError as e: + print(f"Invalid JSON at line {line_num}: {e}") + return data + + def _load_json(self, file_path: Path, config: InstructConfig) -> List[Dict]: + """Load JSON file""" + with open(file_path, 'r', encoding=config.encoding) as f: + data = json.load(f) + + if isinstance(data, list): + return data + elif isinstance(data, dict) and "data" in data: + return data["data"] + else: + return [data] + + def preprocess(self, data: Dict[str, List[Dict]], config: InstructConfig) -> Dict[str, List[Dict]]: + """Apply preprocessing steps to all splits separately""" + processed_splits = {} + + print(f"=== PREPROCESSING CUSTOM CONVERSATION DATA ===") + + for split_name, split_data in data.items(): + print(f"Processing {split_name} split with {len(split_data)} items...") + + processed_data = [] + processed_count = 0 + skipped_count = 0 + + for i, item in enumerate(split_data): + processed_item = self._preprocess_item(item, config) + if processed_item is not None: + processed_data.append(processed_item) + processed_count += 1 + else: + skipped_count += 1 + if skipped_count <= 3: # Log first few skipped items + print(f"Skipped item {i} from {split_name}: {item}") + + processed_splits[split_name] = processed_data + print(f"{split_name} - Preprocessed {processed_count} samples, skipped {skipped_count} samples") + + return processed_splits + + def _preprocess_item(self, item: Dict, config: InstructConfig) -> Optional[Dict]: + """Preprocess a single conversation item""" + conversation = item.get(config.conversation_field, []) + + if not isinstance(conversation, list) or not conversation: + return None + + # Validate conversation structure + valid_conversation = [] + for turn in conversation: + if not isinstance(turn, dict): + continue + if "role" not in turn or "content" not in turn: + continue + if turn["role"] not in ["user", "assistant", "system"]: + continue + + content = str(turn["content"]).strip() + if len(content) < config.min_length or len(content) > config.max_length: + continue + + if config.clean_text: + content = self._clean_text(content) + + valid_conversation.append({ + "role": turn["role"], + "content": content + }) + + if len(valid_conversation) < 2: # Need at least 2 turns for a conversation + return None + + return {"conversation": valid_conversation} + + def _clean_text(self, text: str) -> str: + """Clean and normalize text""" + if not isinstance(text, str): + return "" + + # Remove extra whitespace + text = re.sub(r'\s+', ' ', text).strip() + return text + +class InstructDataPipeline: + """Main instruction fine-tuning data pipeline""" + + def __init__(self): + self.validator = ConversationValidator() + self.hf_loader = HuggingFaceInstructDataLoader() + self.custom_loader = CustomInstructDataLoader() + + def create_config( + self, + data_source: str, + dataset_name: Optional[str] = None, + data_path: Optional[str] = None, + conversation_field: str = "conversation", + **kwargs + ) -> InstructConfig: + """Create instruction configuration""" + return InstructConfig( + data_source=data_source, + dataset_name=dataset_name, + data_path=data_path, + conversation_field=conversation_field, + **kwargs + ) + + def load_config_from_yaml(self, yaml_path: str) -> InstructConfig: + """Load configuration from YAML file""" + try: + config_dict = load_yaml_config(yaml_path) + + # Create configuration object from YAML data + config = InstructConfig( + data_source=config_dict.get('data_source', 'custom'), + dataset_name=config_dict.get('dataset_name'), + data_path=config_dict.get('data_path'), + data_format=config_dict.get('data_format', 'jsonl'), + conversation_field=config_dict.get('conversation_field', 'conversation'), + max_samples=config_dict.get('max_samples'), + train_split=config_dict.get('train_split', 0.8), + validation_split=config_dict.get('validation_split', 0.1), + test_split=config_dict.get('test_split', 0.1), + clean_text=config_dict.get('clean_text', True), + min_length=config_dict.get('min_length', 10), + max_length=config_dict.get('max_length', 2048), + output_format=config_dict.get('output_format', 'conversation'), + output_dir=config_dict.get('output_dir', './data/processed/instruct'), + hf_split=config_dict.get('hf_split', 'train'), + hf_cache_dir=config_dict.get('hf_cache_dir'), + test_split_from=config_dict.get('test_split_from', 'train'), + val_split_from=config_dict.get('val_split_from', 'train'), + encoding=config_dict.get('encoding', 'utf-8') + ) + + print(f"Configuration loaded from YAML: {yaml_path}") + print(f"Output directory: {config.output_dir}") + print(f"Conversation field: {config.conversation_field}") + + return config + + except Exception as e: + print(f"Error loading configuration from YAML {yaml_path}: {e}") + raise + + def load_and_preprocess(self, config: InstructConfig) -> Tuple[Dict[str, List[Dict]], Dict[str, Any]]: + """Load and preprocess conversation data""" + + print(f"Starting conversation data loading and preprocessing...") + print(f"Data source: {config.data_source}") + + try: + # Load data + if config.data_source == "huggingface": + print("Loading HuggingFace conversation dataset...") + raw_splits = self.hf_loader.load(config) + print("Preprocessing HuggingFace conversation dataset...") + processed_splits = self.hf_loader.preprocess(raw_splits, config) + elif config.data_source == "custom": + print("Loading custom conversation dataset...") + raw_splits = self.custom_loader.load(config) + print("Preprocessing custom conversation dataset...") + processed_splits = self.custom_loader.preprocess(raw_splits, config) + else: + raise ValueError(f"Unsupported data source: {config.data_source}") + + print(f"Conversation data loading and preprocessing completed successfully") + + # Validate processed data + print("Validating processed conversation data...") + is_valid, errors = self.validator.validate_conversation_data(processed_splits, config, is_processed=True) + if not is_valid: + print("Conversation data validation failed:") + for error in errors: + print(f" - {error}") + raise ValueError("Conversation data validation failed") + + print("Conversation data validation passed") + + # Analyze dataset + print("Analyzing conversation dataset...") + analysis = self.validator.analyze_conversation_dataset(processed_splits, config, is_processed=True) + print("Conversation dataset analysis completed") + + return processed_splits, analysis + + except Exception as e: + print(f"Error in load_and_preprocess: {e}") + raise + + def save_data(self, data: Dict[str, List[Dict]], output_dir: str, format: str = "jsonl"): + """Save processed conversation data splits to files""" + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + for split_name, split_data in data.items(): + if format == "jsonl": + output_file = output_path / f"{split_name}.jsonl" + with open(output_file, 'w', encoding='utf-8') as f: + for item in split_data: + f.write(json.dumps(item, ensure_ascii=False) + '\n') + elif format == "json": + output_file = output_path / f"{split_name}.json" + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(split_data, f, ensure_ascii=False, indent=2) + + print(f"Saved {len(split_data)} conversation samples to {output_file}") + + def run_pipeline( + self, + config: InstructConfig, + save_splits: bool = True + ) -> Dict[str, Any]: + """Run complete instruction data pipeline""" + + print("Starting instruction data pipeline...") + + # Load and preprocess data + processed_splits, analysis = self.load_and_preprocess(config) + + # Save data if requested + if save_splits: + output_dir = Path(config.output_dir) + self.save_data(processed_splits, str(output_dir)) + + # Create result summary + result = { + "config": config, + "analysis": analysis, + "splits": { + split_name: len(split_data) for split_name, split_data in processed_splits.items() + }, + "output_format": config.output_format, + "output_dir": config.output_dir, + "data": processed_splits, # Include the actual processed data + } + + print("Instruction data pipeline completed successfully!") + return result + +def load_yaml_config(config_path: str) -> Dict[str, Any]: + """Load and parse YAML configuration file with proper structure handling""" + try: + with open(config_path, 'r', encoding='utf-8') as f: + yaml_data = yaml.safe_load(f) + + # Extract configuration from YAML structure + config_dict = {} + + # Handle task section + if 'task' in yaml_data: + task_data = yaml_data['task'] + config_dict.update({ + 'task_name': task_data.get('name'), + 'task_type': task_data.get('type') + }) + + # Handle data section + if 'data' in yaml_data: + data_config = yaml_data['data'] + config_dict.update({ + 'data_source': data_config.get('source'), + 'dataset_name': data_config.get('dataset_name'), + 'data_path': data_config.get('data_path'), + 'data_format': data_config.get('data_format'), + 'conversation_field': data_config.get('conversation_field'), + 'max_samples': data_config.get('max_samples'), + 'train_split': data_config.get('train_split'), + 'validation_split': data_config.get('validation_split'), + 'test_split': data_config.get('test_split'), + 'clean_text': data_config.get('clean_text'), + 'min_length': data_config.get('min_length'), + 'max_length': data_config.get('max_length'), + 'output_format': data_config.get('output_format'), + 'output_dir': data_config.get('output_dir'), + 'encoding': data_config.get('encoding') + }) + + print(f"Successfully parsed YAML configuration from: {config_path}") + print(f"Extracted {len(config_dict)} configuration parameters") + + return config_dict + + except Exception as e: + print(f"Error loading YAML config from {config_path}: {e}") + raise + +def main(): + """Main function with YAML configuration support""" + + parser = argparse.ArgumentParser(description="Instruction Data Processing Pipeline") + + # YAML configuration + parser.add_argument("--config", type=str, help="Path to YAML configuration file") + + # Data source arguments + parser.add_argument("--data-source", choices=["huggingface", "custom"], help="Data source") + parser.add_argument("--dataset-name", type=str, help="HuggingFace dataset name") + parser.add_argument("--data-path", type=str, help="Path to custom data file") + parser.add_argument("--data-format", choices=["jsonl", "json"], help="Data format") + + # Field mapping + parser.add_argument("--conversation-field", type=str, help="Conversation field name") + + # Data processing + parser.add_argument("--max-samples", type=int, help="Maximum samples to process") + parser.add_argument("--train-split", type=float, help="Training split ratio") + parser.add_argument("--validation-split", type=float, help="Validation split ratio") + parser.add_argument("--test-split", type=float, help="Test split ratio") + + # Output configuration + parser.add_argument("--output-dir", type=str, help="Output directory") + + # Logging + parser.add_argument("--log-level", choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO", help="Logging level") + + args = parser.parse_args() + + # Set up logging + # logging.basicConfig( + # level=getattr(logging, args.log_level), + # format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + # ) + + # Load configuration + config_dict = {} + + # Load YAML config if provided + if args.config: + try: + config_dict = load_yaml_config(args.config) + except Exception as e: + print(f"Error loading YAML config: {e}") + sys.exit(1) + + # Override YAML config with CLI arguments (similar to styling pipeline) + cli_overrides = {} + if args.data_source: + cli_overrides['data_source'] = args.data_source + if args.dataset_name: + cli_overrides['dataset_name'] = args.dataset_name + if args.data_path: + cli_overrides['data_path'] = args.data_path + if args.data_format: + cli_overrides['data_format'] = args.data_format + if args.conversation_field: + cli_overrides['conversation_field'] = args.conversation_field + if args.max_samples: + cli_overrides['max_samples'] = args.max_samples + if args.train_split: + cli_overrides['train_split'] = args.train_split + if args.validation_split: + cli_overrides['validation_split'] = args.validation_split + if args.test_split: + cli_overrides['test_split'] = args.test_split + if args.output_dir: + cli_overrides['output_dir'] = args.output_dir + + # Merge configurations + for key, value in cli_overrides.items(): + if key in config_dict: + print(f"Overriding YAML config '{key}' with CLI value: {value}") + config_dict[key] = value + + # Validate required arguments + if not config_dict.get('data_source'): + parser.error("--data-source is required (either in YAML config or CLI)") + + if config_dict.get('data_source') == "huggingface" and not config_dict.get('dataset_name'): + parser.error("--dataset-name is required for HuggingFace datasets") + + if config_dict.get('data_source') == "custom" and not config_dict.get('data_path'): + parser.error("--data-path is required for custom datasets") + + # Create configuration object + config = InstructConfig( + data_source=config_dict.get('data_source', 'custom'), + dataset_name=config_dict.get('dataset_name'), + data_path=config_dict.get('data_path'), + data_format=config_dict.get('data_format', 'jsonl'), + conversation_field=config_dict.get('conversation_field', 'conversation'), + max_samples=config_dict.get('max_samples'), + train_split=config_dict.get('train_split', 0.8), + validation_split=config_dict.get('validation_split', 0.1), + test_split=config_dict.get('test_split', 0.1), + clean_text=config_dict.get('clean_text', True), + min_length=config_dict.get('min_length', 10), + max_length=config_dict.get('max_length', 2048), + output_format=config_dict.get('output_format', 'conversation'), + output_dir=config_dict.get('output_dir', './data/processed/instruct'), + hf_split=config_dict.get('hf_split', 'train'), + hf_cache_dir=config_dict.get('hf_cache_dir'), + test_split_from=config_dict.get('test_split_from', 'train'), + val_split_from=config_dict.get('val_split_from', 'train'), + encoding=config_dict.get('encoding', 'utf-8') + ) + + # Initialize pipeline + pipeline = InstructDataPipeline() + + try: + print(f"Starting instruction data pipeline with {config.data_source} data source...") + if args.config: + print(f"Using YAML configuration: {args.config}") + print(f"Conversation field: {config.conversation_field}") + print() + + result = pipeline.run_pipeline(config, save_splits=True) + + print(f"✅ Pipeline completed successfully!") + print(f" Data source: {config.data_source}") + if config.data_source == "huggingface": + print(f" Dataset: {config.dataset_name}") + else: + print(f" Data file: {config.data_path}") + print(f" Total samples: {result['analysis']['overall']['total_samples']}") + print(f" Split sizes: {result['analysis']['overall']['split_sizes']}") + print(f" Output directory: {config.output_dir}") + print(f" Conversation stats: {result['analysis']['overall']['conversation_stats']}") + + except Exception as e: + print(f"❌ Error running pipeline: {e}") + import traceback + print("Full error traceback:") + traceback.print_exc() + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/pipelines/instruct/.ipynb_checkpoints/inference-checkpoint.py b/pipelines/instruct/.ipynb_checkpoints/inference-checkpoint.py index e69de29..161a7a4 100644 --- a/pipelines/instruct/.ipynb_checkpoints/inference-checkpoint.py +++ b/pipelines/instruct/.ipynb_checkpoints/inference-checkpoint.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +""" +Instruct Inference Pipeline using Trained Models +Supports conversational inference with streaming and batch processing +""" + +import os +import sys +import json +import argparse +from pathlib import Path +from typing import Dict, Any, Optional, List, Union +import yaml + +# Add the project root to the path +sys.path.append(str(Path(__file__).parent.parent.parent)) + +# Inference imports +import torch +from datasets import load_from_disk, Dataset +from unsloth import FastLanguageModel +from unsloth.chat_templates import get_chat_template +from transformers import TextStreamer + +class InstructInference: + """Instruction fine-tuning inference using trained models""" + + def __init__(self, config: Dict[str, Any]): + self.config = config + self.model = None + self.tokenizer = None + + # Set device + self.device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using device: {self.device}") + + # Model parameters + self.model_output_dir = config.get('model_output_dir', './models/instruct') + self.base_model_name = config.get('base_model_name', 'unsloth/Qwen2.5-72B-Instruct') + self.max_seq_length = config.get('max_seq_length', 2048) + self.dtype = config.get('dtype', None) + self.load_in_4bit = config.get('load_in_4bit', True) + self.hf_token = config.get('hf_token', None) + + # Inference parameters + self.batch_size = config.get('batch_size', 1) + self.max_new_tokens = config.get('max_new_tokens', 128) + self.temperature = config.get('temperature', 1.5) + self.min_p = config.get('min_p', 0.1) + self.use_cache = config.get('use_cache', True) + + # Chat template + self.chat_template = config.get('chat_template', 'qwen-2.5') + + def load_model_and_tokenizer(self): + """Load the trained model and tokenizer""" + print("Loading trained instruction model and tokenizer...") + + try: + # Load the saved LoRA model + model_path = self.model_output_dir + print(f"Loading model from: {model_path}") + + # Check if the model directory exists + if not Path(model_path).exists(): + raise FileNotFoundError(f"Model directory not found: {model_path}") + + # Load the model directly from the saved path + self.model, self.tokenizer = FastLanguageModel.from_pretrained( + model_name=model_path, + max_seq_length=self.max_seq_length, + dtype=self.dtype, + load_in_4bit=self.load_in_4bit, + ) + + # Enable native 2x faster inference + FastLanguageModel.for_inference(self.model) + + print(f"✅ Model loaded from: {model_path}") + print(f"✅ Tokenizer loaded with vocab size: {self.tokenizer.vocab_size}") + + except Exception as e: + print(f"❌ Error loading model: {e}") + raise + + def setup_chat_template(self): + """Setup chat template for conversation formatting""" + print("Setting up chat template...") + + try: + self.tokenizer = get_chat_template( + self.tokenizer, + chat_template=self.chat_template, + ) + + print(f"✅ Chat template configured: {self.chat_template}") + + except Exception as e: + print(f"❌ Error setting up chat template: {e}") + raise + + def format_messages(self, messages: List[Dict[str, str]]) -> str: + """Format messages using chat template""" + try: + # Apply chat template to format the conversation + formatted_prompt = self.tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, # Add generation prompt for inference + ) + + return formatted_prompt + + except Exception as e: + print(f"❌ Error formatting messages: {e}") + raise + + def generate_response( + self, + messages: List[Dict[str, str]], + max_new_tokens: Optional[int] = None, + temperature: Optional[float] = None, + stream: bool = False + ) -> str: + """Generate response using the trained instruction model""" + try: + # Use default values if not provided + max_tokens = max_new_tokens or self.max_new_tokens + temp = temperature or self.temperature + + # Format the messages + formatted_prompt = self.format_messages(messages) + print(f"Formatted prompt: {formatted_prompt[:200]}...") + + # Tokenize the input + inputs = self.tokenizer( + [formatted_prompt], + return_tensors="pt" + ).to(self.device) + + if stream: + # Streaming generation + text_streamer = TextStreamer(self.tokenizer, skip_prompt=True) + print("Generating with streaming...") + _ = self.model.generate( + input_ids=inputs.input_ids, + streamer=text_streamer, + max_new_tokens=max_tokens, + use_cache=self.use_cache, + temperature=temp, + min_p=self.min_p + ) + return "" # Streaming output is handled by streamer + else: + # Non-streaming generation + print("Generating response...") + outputs = self.model.generate( + input_ids=inputs.input_ids, + max_new_tokens=max_tokens, + use_cache=self.use_cache, + temperature=temp, + min_p=self.min_p, + pad_token_id=self.tokenizer.eos_token_id + ) + + # Decode the generated text + full_response = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)[0] + + # Extract only the new generated response (remove the input prompt) + prompt_length = len(formatted_prompt) + response = full_response[prompt_length:].strip() + + return response + + except Exception as e: + print(f"❌ Error generating response: {e}") + raise + + def chat(self, user_input: str, conversation_history: Optional[List[Dict[str, str]]] = None, stream: bool = False) -> str: + """Have a chat conversation with the model""" + try: + # Initialize conversation history if not provided + if conversation_history is None: + conversation_history = [] + + # Add user input to conversation + messages = conversation_history + [{"role": "user", "content": user_input}] + + print(f"User: {user_input}") + + if stream: + print("Assistant: ", end="", flush=True) + self.generate_response(messages, stream=True) + return "" + else: + # Generate response + response = self.generate_response(messages, stream=False) + print(f"Assistant: {response}") + return response + + except Exception as e: + print(f"❌ Error in chat: {e}") + raise + + def batch_inference( + self, + conversations: List[List[Dict[str, str]]], + max_new_tokens: Optional[int] = None + ) -> List[str]: + """Perform batch inference on multiple conversations""" + responses = [] + + for i, messages in enumerate(conversations): + print(f"Processing conversation {i+1}/{len(conversations)}") + response = self.generate_response(messages, max_new_tokens) + responses.append(response) + + return responses + + def interactive_chat(self): + """Start an interactive chat session""" + print("🤖 Starting interactive chat session...") + print("Type 'quit', 'exit', or 'bye' to end the conversation.") + print("Type 'clear' to clear conversation history.") + print("Type 'stream on' or 'stream off' to toggle streaming.") + print("-" * 50) + + conversation_history = [] + streaming = False + + while True: + try: + user_input = input("\n👤 You: ").strip() + + if user_input.lower() in ['quit', 'exit', 'bye']: + print("👋 Goodbye!") + break + elif user_input.lower() == 'clear': + conversation_history = [] + print("🗑️ Conversation history cleared.") + continue + elif user_input.lower() == 'stream on': + streaming = True + print("🔄 Streaming enabled.") + continue + elif user_input.lower() == 'stream off': + streaming = False + print("⏸️ Streaming disabled.") + continue + elif not user_input: + continue + + # Generate response + if streaming: + print("🤖 Assistant: ", end="", flush=True) + self.chat(user_input, conversation_history, stream=True) + # Add to history (we don't have the actual response text for streaming) + conversation_history.extend([ + {"role": "user", "content": user_input}, + {"role": "assistant", "content": "[Streamed response]"} + ]) + else: + response = self.chat(user_input, conversation_history, stream=False) + # Add to history + conversation_history.extend([ + {"role": "user", "content": user_input}, + {"role": "assistant", "content": response} + ]) + + except KeyboardInterrupt: + print("\n👋 Goodbye!") + break + except Exception as e: + print(f"❌ Error: {e}") + continue + +def load_inference_config(config_path: str) -> Dict[str, Any]: + """Load inference configuration from YAML file""" + try: + with open(config_path, 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) + + # Extract inference configuration + inference_config = {} + + # Model configuration + if 'model' in config: + model_data = config['model'] + inference_config.update({ + 'base_model_name': model_data.get('training_model', 'unsloth/Qwen2.5-72B-Instruct'), + 'max_seq_length': model_data.get('training_max_seq_length', 2048), + 'dtype': model_data.get('training_dtype'), + 'load_in_4bit': model_data.get('training_load_in_4bit', True), + 'hf_token': model_data.get('training_token') + }) + + # Training configuration - to get model_output_dir + if 'training' in config: + training_data = config['training'] + inference_config.update({ + 'model_output_dir': training_data.get('model_output_dir', './models/instruct') + }) + + # Inference configuration + if 'inference' in config: + inference_data = config['inference'] + inference_config.update({ + 'batch_size': inference_data.get('batch_size', 1), + 'max_new_tokens': inference_data.get('max_new_tokens', 128), + 'temperature': inference_data.get('temperature', 1.5), + 'min_p': inference_data.get('min_p', 0.1), + 'use_cache': inference_data.get('use_cache', True) + }) + + # Chat template + inference_config.update({ + 'chat_template': 'qwen-2.5' # Use Qwen chat template by default + }) + + return inference_config + + except Exception as e: + print(f"Error loading inference config: {e}") + raise + +def main(): + """Main inference function""" + parser = argparse.ArgumentParser(description="Instruction Inference Pipeline") + + # Configuration + parser.add_argument("--config", type=str, required=True, help="Path to YAML configuration file") + parser.add_argument("--interactive", action="store_true", help="Start interactive chat session") + parser.add_argument("--message", type=str, help="Single message to send to the model") + parser.add_argument("--max-tokens", type=int, help="Maximum new tokens to generate") + parser.add_argument("--stream", action="store_true", help="Enable streaming generation") + parser.add_argument("--temperature", type=float, help="Sampling temperature") + + args = parser.parse_args() + + try: + # Load configuration + print(f"Loading configuration from: {args.config}") + inference_config = load_inference_config(args.config) + + # Override with CLI arguments + if args.max_tokens: + inference_config['max_new_tokens'] = args.max_tokens + if args.temperature: + inference_config['temperature'] = args.temperature + + print("Inference configuration:") + for key, value in inference_config.items(): + print(f" {key}: {value}") + + # Initialize inference + inference = InstructInference(inference_config) + + # Load model and tokenizer + inference.load_model_and_tokenizer() + + # Setup chat template + inference.setup_chat_template() + + # Run inference based on mode + if args.interactive: + # Interactive chat mode + inference.interactive_chat() + elif args.message: + # Single message mode + print("Running single message inference...") + messages = [{"role": "user", "content": args.message}] + + if args.stream: + print("User:", args.message) + print("Assistant: ", end="", flush=True) + inference.generate_response(messages, stream=True) + else: + response = inference.generate_response(messages, stream=False) + print(f"User: {args.message}") + print(f"Assistant: {response}") + else: + # Default to interactive mode if no specific mode is chosen + print("No specific mode chosen. Starting interactive chat...") + inference.interactive_chat() + + except Exception as e: + print(f"Inference failed: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/pipelines/instruct/.ipynb_checkpoints/train-checkpoint.py b/pipelines/instruct/.ipynb_checkpoints/train-checkpoint.py index e69de29..c37dd0c 100644 --- a/pipelines/instruct/.ipynb_checkpoints/train-checkpoint.py +++ b/pipelines/instruct/.ipynb_checkpoints/train-checkpoint.py @@ -0,0 +1,518 @@ +#!/usr/bin/env python3 +""" +Instruct Training Pipeline using Unsloth and SFTTrainer +Supports instruction fine-tuning with conversational data and LoRA fine-tuning +""" + +import os +import sys +import json +import argparse +from pathlib import Path +from typing import Dict, Any, Optional, List +import yaml + +# Add the project root to the path +sys.path.append(str(Path(__file__).parent.parent.parent)) + +from utils.config.config_manager import ConfigManager + +# Training imports +import torch +from datasets import load_from_disk, Dataset +from unsloth import FastLanguageModel #is_bfloat16_supported +from unsloth.chat_templates import get_chat_template, standardize_sharegpt, train_on_responses_only +from trl import SFTTrainer, SFTConfig +from transformers import DataCollatorForSeq2Seq + +class InstructTrainer: + """Instruction fine-tuning trainer using Unsloth and SFTTrainer""" + + def __init__(self, config: Dict[str, Any]): + self.config = config + self.model = None + self.tokenizer = None + self.trainer = None + + # Set device + self.device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using device: {self.device}") + + # Model parameters + self.model_name = config.get('model_name', 'unsloth/Qwen2.5-72B-Instruct') + self.max_seq_length = config.get('max_seq_length', 2048) + self.dtype = config.get('dtype', None) + self.load_in_4bit = config.get('load_in_4bit', True) + self.hf_token = config.get('hf_token', None) + + # LoRA parameters + self.lora_r = config.get('lora_r', 32) + self.lora_alpha = config.get('lora_alpha', 16) + self.lora_dropout = config.get('lora_dropout', 0) + self.target_modules = config.get('target_modules', [ + "q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj" + ]) + + # Training arguments + self.batch_size = config.get('batch_size', 1) + self.gradient_accumulation_steps = config.get('gradient_accumulation_steps', 4) + self.learning_rate = config.get('learning_rate', 2e-4) + self.num_epochs = config.get('num_epochs', 1) + self.max_steps = config.get('max_steps', 30) + self.warmup_steps = config.get('warmup_steps', 5) + self.weight_decay = config.get('weight_decay', 0.01) + self.seed = config.get('seed', 3407) + + # Output paths + self.output_dir = config.get('output_dir', './outputs') + self.model_output_dir = config.get('model_output_dir', './models/instruct') + + # Chat template + self.chat_template = config.get('chat_template', 'qwen-2.5') + + def load_model_and_tokenizer(self): + """Load the pre-trained model and tokenizer""" + print("Loading model and tokenizer...") + + try: + self.model, self.tokenizer = FastLanguageModel.from_pretrained( + model_name=self.model_name, + max_seq_length=self.max_seq_length, + dtype=self.dtype, + load_in_4bit=self.load_in_4bit, + token=self.hf_token + ) + + print(f"✅ Model loaded: {self.model_name}") + print(f"✅ Tokenizer loaded with vocab size: {self.tokenizer.vocab_size}") + + except Exception as e: + print(f"❌ Error loading model: {e}") + raise + + def setup_lora(self): + """Setup LoRA for efficient fine-tuning""" + print("Setting up LoRA configuration...") + + try: + self.model = FastLanguageModel.get_peft_model( + self.model, + r=self.lora_r, + target_modules=self.target_modules, + lora_alpha=self.lora_alpha, + lora_dropout=self.lora_dropout, + bias="none", + use_gradient_checkpointing="unsloth", + random_state=self.seed, + use_rslora=False, + loftq_config=None + ) + + print(f"✅ LoRA configured with r={self.lora_r}, alpha={self.lora_alpha}") + + except Exception as e: + print(f"❌ Error setting up LoRA: {e}") + raise + + def setup_chat_template(self): + """Setup chat template for conversation formatting""" + print("Setting up chat template...") + + try: + self.tokenizer = get_chat_template( + self.tokenizer, + chat_template=self.chat_template, + ) + + print(f"✅ Chat template configured: {self.chat_template}") + + except Exception as e: + print(f"❌ Error setting up chat template: {e}") + raise + + def load_dataset(self, dataset_path: str) -> Dataset: + """Load the conversation training dataset""" + print(f"Loading conversation dataset from: {dataset_path}") + + try: + if Path(dataset_path).exists(): + # Check if it's a HuggingFace dataset directory + if (Path(dataset_path) / "dataset_info.json").exists(): + # Load from HuggingFace dataset directory + dataset = load_from_disk(dataset_path) + print(f"Loaded HuggingFace dataset from disk: {len(dataset)} samples") + else: + # Load from processed conversation data files (JSONL format) + print("Loading from processed conversation data files...") + from datasets import Dataset + import json + + all_data = [] + data_dir = Path(dataset_path) + + # Look for train.jsonl, validation.jsonl, test.jsonl + for split_file in ["train.jsonl", "validation.jsonl", "test.jsonl"]: + file_path = data_dir / split_file + if file_path.exists(): + print(f"Loading {split_file}...") + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + if line.strip(): + data = json.loads(line) + all_data.append(data) + + if not all_data: + raise ValueError(f"No conversation data found in {dataset_path}") + + # Create HuggingFace dataset + dataset = Dataset.from_list(all_data) + print(f"Created HuggingFace dataset from {len(all_data)} conversation samples") + else: + # Try loading from HuggingFace Hub + print(f"Attempting to load from HuggingFace Hub: {dataset_path}") + dataset = Dataset.load_dataset(dataset_path, split="train") + print(f"Loaded from HuggingFace Hub: {len(dataset)} samples") + + print(f"Dataset loaded: {len(dataset)} samples") + print(f"Dataset features: {dataset.features}") + + # Verify required fields exist for conversation data + required_fields = ["conversation"] + missing_fields = [field for field in required_fields if field not in dataset.features] + if missing_fields: + raise ValueError(f"Missing required fields in conversation dataset: {missing_fields}") + + return dataset + + except Exception as e: + print(f"Error loading conversation dataset: {e}") + raise + + def format_dataset_for_training(self, dataset: Dataset) -> Dataset: + """Format conversation dataset for training using standardize_sharegpt and apply_chat_template""" + print("Formatting conversation dataset for training...") + + try: + # Standardize the ShareGPT format + print("Standardizing ShareGPT format...") + dataset = standardize_sharegpt(dataset) + + # Define the formatting function for chat templates + def formatting_prompts_func(examples): + convos = examples["conversation"] + texts = [ + self.tokenizer.apply_chat_template( + convo, + tokenize=False, + add_generation_prompt=False + ) for convo in convos + ] + return {"text": texts} + + # Apply the formatting function + print("Applying chat template formatting...") + dataset = dataset.map(formatting_prompts_func, batched=True) + + print(f"✅ Dataset formatted for training with {len(dataset)} samples") + print(f"Sample formatted text: {dataset[0]['text'][:200]}...") + + return dataset + + except Exception as e: + print(f"❌ Error formatting dataset: {e}") + raise + + def setup_trainer(self, train_dataset: Dataset): + """Setup the SFTTrainer for instruction fine-tuning""" + print("Setting up SFTTrainer for instruction fine-tuning...") + + try: + # SFT Configuration + sft_config = SFTConfig( + per_device_train_batch_size=self.batch_size, + gradient_accumulation_steps=self.gradient_accumulation_steps, + warmup_steps=self.warmup_steps, + max_steps=self.max_steps, + learning_rate=self.learning_rate, + logging_steps=1, + optim="paged_adamw_8bit", + weight_decay=self.weight_decay, + lr_scheduler_type="linear", + seed=self.seed, + output_dir=self.output_dir, + report_to="none", # Disable wandb for now + ) + + print("SFT Configuration:") + print(f" batch_size: {self.batch_size}") + print(f" gradient_accumulation_steps: {self.gradient_accumulation_steps}") + print(f" warmup_steps: {self.warmup_steps}") + print(f" max_steps: {self.max_steps}") + print(f" learning_rate: {self.learning_rate}") + + # Create SFTTrainer + self.trainer = SFTTrainer( + model=self.model, + tokenizer=self.tokenizer, + train_dataset=train_dataset, + dataset_text_field="text", + max_seq_length=self.max_seq_length, + data_collator=DataCollatorForSeq2Seq(tokenizer=self.tokenizer), + packing=False, # Disable packing for conversation data + args=sft_config, + ) + + print("✅ SFTTrainer configured successfully") + + except Exception as e: + print(f"❌ Error setting up trainer: {e}") + import traceback + print("Full error traceback:") + traceback.print_exc() + raise + + def setup_response_only_training(self): + """Setup training to only learn from assistant responses""" + print("Setting up response-only training...") + + try: + # Configure trainer to only train on responses + self.trainer = train_on_responses_only( + self.trainer, + instruction_part="<|im_start|>user\n", + response_part="<|im_start|>assistant\n", + ) + + print("✅ Response-only training configured") + + except Exception as e: + print(f"❌ Error setting up response-only training: {e}") + raise + + def train(self, dataset_path: str): + """Run the instruction fine-tuning process""" + print("🚀 Starting instruction fine-tuning process...") + + try: + # Load model and tokenizer + print("Step 1: Loading model and tokenizer...") + self.load_model_and_tokenizer() + + # Setup LoRA + print("Step 2: Setting up LoRA...") + self.setup_lora() + + # Setup chat template + print("Step 3: Setting up chat template...") + self.setup_chat_template() + + # Load dataset + print(f"Step 4: Loading conversation dataset from: {dataset_path}") + train_dataset = self.load_dataset(dataset_path) + + # Format dataset for training + print("Step 5: Formatting dataset for training...") + formatted_dataset = self.format_dataset_for_training(train_dataset) + + # Setup trainer + print("Step 6: Setting up trainer...") + self.setup_trainer(formatted_dataset) + + # Setup response-only training (optional but recommended for chat models) + print("Step 7: Setting up response-only training...") + self.setup_response_only_training() + + # Start training + print("Step 8: Starting training...") + trainer_stats = self.trainer.train() + + print("✅ Instruction fine-tuning completed successfully!") + print(f"Training stats: {trainer_stats}") + + # Save the model + self.save_model() + + return trainer_stats + + except Exception as e: + print(f"❌ Instruction fine-tuning failed: {e}") + import traceback + print("Full error traceback:") + traceback.print_exc() + raise + + def save_model(self): + """Save the trained instruction model""" + print("Saving trained instruction model...") + + try: + # Create output directory + Path(self.model_output_dir).mkdir(parents=True, exist_ok=True) + + # Save model and tokenizer + self.model.save_pretrained(self.model_output_dir) + self.tokenizer.save_pretrained(self.model_output_dir) + + # Save training config + config_path = Path(self.model_output_dir) / "training_config.json" + with open(config_path, 'w') as f: + json.dump(self.config, f, indent=2) + + print(f"✅ Instruction model saved to: {self.model_output_dir}") + print(f"✅ You can now use this model for inference") + + except Exception as e: + print(f"❌ Error saving model: {e}") + raise + + def prepare_for_inference(self): + """Prepare model for inference""" + print("Preparing model for inference...") + + try: + FastLanguageModel.for_inference(self.model) + print("✅ Model prepared for inference") + + except Exception as e: + print(f"❌ Error preparing for inference: {e}") + raise + +def load_training_config(yaml_path: str) -> Dict[str, Any]: + """Load training configuration from YAML file""" + try: + with open(yaml_path, 'r') as f: + config = yaml.safe_load(f) + + training_config = {} + + # Model configuration - extract from model section + if 'model' in config: + model_config = config['model'] + training_config.update({ + 'model_name': model_config.get('name', 'unsloth/Qwen2.5-72B-Instruct'), + 'max_seq_length': int(model_config.get('max_seq_length', 2048)), + 'dtype': model_config.get('dtype', None), + 'load_in_4bit': model_config.get('load_in_4bit', True), + 'hf_token': model_config.get('token', None) + }) + + # Training configuration - extract from training section + if 'training' in config: + training_data = config['training'] + print("Training data from YAML:") + print(f" num_epochs: {training_data.get('num_epochs')} (type: {type(training_data.get('num_epochs'))})") + print(f" batch_size: {training_data.get('batch_size')} (type: {type(training_data.get('batch_size'))})") + print(f" learning_rate: {training_data.get('learning_rate')} (type: {type(training_data.get('learning_rate'))})") + print(f" weight_decay: {training_data.get('weight_decay')} (type: {type(training_data.get('weight_decay'))})") + print(f" warmup_steps: {training_data.get('warmup_steps')} (type: {type(training_data.get('warmup_steps'))})") + print(f" max_steps: {training_data.get('max_steps')} (type: {type(training_data.get('max_steps'))})") + print(f" gradient_accumulation_steps: {training_data.get('gradient_accumulation_steps')} (type: {type(training_data.get('gradient_accumulation_steps'))})") + print(f" seed: {training_data.get('seed')} (type: {type(training_data.get('seed'))})") + print(f" model_output_dir: {training_data.get('model_output_dir')} (type: {type(training_data.get('model_output_dir'))})") + + training_config.update({ + 'num_epochs': int(training_data.get('num_epochs', 1)), + 'batch_size': int(training_data.get('batch_size', 1)), + 'learning_rate': float(training_data.get('learning_rate', 2e-4)), + 'weight_decay': float(training_data.get('weight_decay', 0.01)), + 'warmup_steps': int(training_data.get('warmup_steps', 5)), + 'max_steps': int(training_data.get('max_steps', 30)), + 'gradient_accumulation_steps': int(training_data.get('gradient_accumulation_steps', 4)), + 'lr_scheduler_type': training_data.get('lr_scheduler_type', 'linear'), + 'seed': int(training_data.get('seed', 3407)), + 'model_output_dir': training_data.get('model_output_dir', './models/instruct'), + # LoRA configuration + 'lora_r': int(training_data.get('lora_r', 32)), + 'lora_alpha': int(training_data.get('lora_alpha', 16)), + 'lora_dropout': float(training_data.get('lora_dropout', 0)), + 'target_modules': training_data.get('target_modules', [ + "q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj" + ]) + }) + + # Data configuration - use output_dir from data section + if 'data' in config: + data_config = config['data'] + output_dir = data_config.get('output_dir', './data/processed/instruct') + training_config.update({ + 'data_output_dir': output_dir, + 'dataset_path': output_dir, # Default dataset path is the output_dir + }) + + # Output configuration + training_config.update({ + 'output_dir': './outputs', + 'chat_template': 'qwen-2.5' # Use Qwen chat template by default + }) + + print("Final training_config:") + for key, value in training_config.items(): + print(f" {key}: {value} (type: {type(value)})") + + return training_config + + except Exception as e: + print(f"Error loading training config: {e}") + raise + +def main(): + """Main training function""" + parser = argparse.ArgumentParser(description="Instruction Fine-tuning Training Pipeline") + + # Configuration + parser.add_argument("--config", type=str, required=True, help="Path to YAML configuration file") + parser.add_argument("--dataset", type=str, help="Path to training dataset (conversation data path)") + parser.add_argument("--output-dir", type=str, help="Output directory for model") + parser.add_argument("--epochs", type=int, help="Number of training epochs") + parser.add_argument("--batch-size", type=int, help="Training batch size") + parser.add_argument("--learning-rate", type=float, help="Learning rate") + parser.add_argument("--max-steps", type=int, help="Maximum training steps") + + args = parser.parse_args() + + # Setup logging replaced with print statements + + try: + # Load configuration + print(f"Loading configuration from: {args.config}") + training_config = load_training_config(args.config) + + # Override with CLI arguments + if args.output_dir: + training_config['model_output_dir'] = args.output_dir + if args.epochs: + training_config['num_epochs'] = int(args.epochs) + if args.batch_size: + training_config['batch_size'] = int(args.batch_size) + if args.learning_rate: + training_config['learning_rate'] = float(args.learning_rate) + if args.max_steps: + training_config['max_steps'] = int(args.max_steps) + + # Determine dataset path: CLI argument takes precedence, then YAML config + dataset_path = args.dataset or training_config.get('dataset_path') + if not dataset_path: + print("No dataset path provided. Use --dataset or ensure output_dir is set in YAML config.") + sys.exit(1) + + print("Training configuration:") + for key, value in training_config.items(): + print(f" {key}: {value}") + print(f" Dataset path: {dataset_path}") + + # Initialize trainer + trainer = InstructTrainer(training_config) + + # Start training + trainer.train(dataset_path) + + print("Instruction fine-tuning completed successfully!") + + except Exception as e: + print(f"Instruction fine-tuning failed: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/pipelines/instruct/data_processor.py b/pipelines/instruct/data_processor.py index 10acda3..30a912c 100644 --- a/pipelines/instruct/data_processor.py +++ b/pipelines/instruct/data_processor.py @@ -7,16 +7,12 @@ from datasets import Dataset, load_dataset import os from dataclasses import dataclass from abc import ABC, abstractmethod -import logging from sklearn.model_selection import train_test_split import re import argparse import sys import yaml -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - @dataclass class InstructConfig: """Configuration for instruction fine-tuning tasks""" @@ -75,7 +71,7 @@ class ConversationValidator: return False, errors total_samples = sum(len(split_data) for split_data in data.values()) - logger.info(f"Validating {total_samples} total samples across all splits...") + print(f"Validating {total_samples} total samples across all splits...") # Determine field names based on whether data is processed or not conversation_field = "conversation" if not is_processed else "conversation" @@ -83,10 +79,10 @@ class ConversationValidator: # Validate each split for split_name, split_data in data.items(): if not split_data: - logger.info(f"Skipping validation for empty {split_name} split") + print(f"Skipping validation for empty {split_name} split") continue - logger.info(f"Validating {split_name} split with {len(split_data)} samples...") + print(f"Validating {split_name} split with {len(split_data)} samples...") # Check required fields missing_conversation_count = 0 @@ -117,19 +113,19 @@ class ConversationValidator: if "role" in turn and turn["role"] not in ["user", "assistant", "system"]: errors.append(f"Invalid role '{turn['role']}' in conversation turn {j}, {split_name} split, item {i}. Must be 'user', 'assistant', or 'system'") - logger.info(f"{split_name} - Items missing conversation field: {missing_conversation_count}") + print(f"{split_name} - Items missing conversation field: {missing_conversation_count}") # Show sample of processed data for debugging if split_data: - logger.info(f"Sample conversation from {split_name}:") + print(f"Sample conversation from {split_name}:") for i in range(min(2, len(split_data))): item = split_data[i] conversation = item.get(conversation_field, []) - logger.info(f" Item {i} conversation length: {len(conversation)} turns") + print(f" Item {i} conversation length: {len(conversation)} turns") for j, turn in enumerate(conversation[:3]): # Show first 3 turns role = turn.get("role", "unknown") content = turn.get("content", "")[:100] + "..." if len(turn.get("content", "")) > 100 else turn.get("content", "") - logger.info(f" Turn {j}: {role} -> '{content}'") + print(f" Turn {j}: {role} -> '{content}'") return len(errors) == 0, errors @@ -242,7 +238,7 @@ class HuggingFaceInstructDataLoader(BaseInstructDataLoader): if not config.dataset_name: raise ValueError("Dataset name is required for Hugging Face datasets") - logger.info(f"Loading Hugging Face conversation dataset: {config.dataset_name}") + print(f"Loading Hugging Face conversation dataset: {config.dataset_name}") try: dataset = load_dataset( @@ -251,7 +247,7 @@ class HuggingFaceInstructDataLoader(BaseInstructDataLoader): ) available_splits = list(dataset.keys()) - logger.info(f"Available splits in dataset: {available_splits}") + print(f"Available splits in dataset: {available_splits}") splits_data = { "train": [], @@ -262,10 +258,10 @@ class HuggingFaceInstructDataLoader(BaseInstructDataLoader): # Handle train split if "train" in available_splits: train_dataset = dataset["train"] - logger.info(f"Using 'train' split with {len(train_dataset)} samples") + print(f"Using 'train' split with {len(train_dataset)} samples") splits_data["train"] = list(train_dataset) else: - logger.error("No 'train' split found in dataset!") + print("No 'train' split found in dataset!") raise ValueError(f"Dataset {config.dataset_name} does not have a 'train' split") # Handle validation and test splits (similar logic to styling pipeline) @@ -277,23 +273,23 @@ class HuggingFaceInstructDataLoader(BaseInstructDataLoader): if splits_data[split_name]: original_size = len(splits_data[split_name]) splits_data[split_name] = splits_data[split_name][:config.max_samples] - logger.info(f"Limited {split_name} split from {original_size} to {len(splits_data[split_name])} samples") + print(f"Limited {split_name} split from {original_size} to {len(splits_data[split_name])} samples") - logger.info(f"Successfully loaded dataset {config.dataset_name}") + print(f"Successfully loaded dataset {config.dataset_name}") return splits_data except Exception as e: - logger.error(f"Error loading dataset {config.dataset_name}: {e}") + print(f"Error loading dataset {config.dataset_name}: {e}") raise def preprocess(self, data: Dict[str, List[Dict]], config: InstructConfig) -> Dict[str, List[Dict]]: """Apply preprocessing steps to all splits separately""" processed_splits = {} - logger.info(f"=== PREPROCESSING CONVERSATION DATA ===") + print(f"=== PREPROCESSING CONVERSATION DATA ===") for split_name, split_data in data.items(): - logger.info(f"Processing {split_name} split with {len(split_data)} items...") + print(f"Processing {split_name} split with {len(split_data)} items...") processed_data = [] processed_count = 0 @@ -308,7 +304,7 @@ class HuggingFaceInstructDataLoader(BaseInstructDataLoader): skipped_count += 1 processed_splits[split_name] = processed_data - logger.info(f"{split_name} - Preprocessed {processed_count} samples, skipped {skipped_count} samples") + print(f"{split_name} - Preprocessed {processed_count} samples, skipped {skipped_count} samples") return processed_splits @@ -368,7 +364,7 @@ class CustomInstructDataLoader(BaseInstructDataLoader): if not file_path.exists(): raise FileNotFoundError(f"Data file not found: {file_path}") - logger.info(f"Loading custom conversation dataset: {file_path}") + print(f"Loading custom conversation dataset: {file_path}") if config.data_format == "jsonl": raw_data = self._load_jsonl(file_path, config) @@ -380,7 +376,7 @@ class CustomInstructDataLoader(BaseInstructDataLoader): if config.max_samples: raw_data = raw_data[:config.max_samples] - logger.info(f"Loaded {len(raw_data)} conversation samples from {file_path}") + print(f"Loaded {len(raw_data)} conversation samples from {file_path}") # Create splits from the raw data splits_data = self._create_splits(raw_data, config) @@ -389,11 +385,11 @@ class CustomInstructDataLoader(BaseInstructDataLoader): def _create_splits(self, data: List[Dict], config: InstructConfig) -> Dict[str, List[Dict]]: """Create train/validation/test splits from raw data""" - logger.info(f"Creating splits from {len(data)} conversation samples...") + print(f"Creating splits from {len(data)} conversation samples...") # Handle very small datasets if len(data) < 3: - logger.warning(f"Dataset has only {len(data)} samples. Using all data for training.") + print(f"Dataset has only {len(data)} samples. Using all data for training.") return { "train": data, "validation": [], @@ -408,7 +404,7 @@ class CustomInstructDataLoader(BaseInstructDataLoader): config.train_split = 0.6 config.validation_split = 0.2 config.test_split = 0.2 - logger.info(f"Small dataset detected. Adjusted split ratios to: train={config.train_split}, val={config.validation_split}, test={config.test_split}") + print(f"Small dataset detected. Adjusted split ratios to: train={config.train_split}, val={config.validation_split}, test={config.test_split}") val_size = max(1, int(total_samples * config.validation_split)) test_size = max(1, int(total_samples * config.test_split)) @@ -423,7 +419,7 @@ class CustomInstructDataLoader(BaseInstructDataLoader): test_size -= 1 train_size += 1 - logger.info(f"Split sizes: train={train_size}, validation={val_size}, test={test_size}") + print(f"Split sizes: train={train_size}, validation={val_size}, test={test_size}") # Create splits if val_size == 0 and test_size == 0: @@ -466,10 +462,10 @@ class CustomInstructDataLoader(BaseInstructDataLoader): "test": test_data } - logger.info(f"Created conversation splits:") - logger.info(f" Train: {len(splits_data['train'])} samples") - logger.info(f" Validation: {len(splits_data['validation'])} samples") - logger.info(f" Test: {len(splits_data['test'])} samples") + print(f"Created conversation splits:") + print(f" Train: {len(splits_data['train'])} samples") + print(f" Validation: {len(splits_data['validation'])} samples") + print(f" Test: {len(splits_data['test'])} samples") return splits_data @@ -482,7 +478,7 @@ class CustomInstructDataLoader(BaseInstructDataLoader): try: data.append(json.loads(line)) except json.JSONDecodeError as e: - logger.warning(f"Invalid JSON at line {line_num}: {e}") + print(f"Invalid JSON at line {line_num}: {e}") return data def _load_json(self, file_path: Path, config: InstructConfig) -> List[Dict]: @@ -501,10 +497,10 @@ class CustomInstructDataLoader(BaseInstructDataLoader): """Apply preprocessing steps to all splits separately""" processed_splits = {} - logger.info(f"=== PREPROCESSING CUSTOM CONVERSATION DATA ===") + print(f"=== PREPROCESSING CUSTOM CONVERSATION DATA ===") for split_name, split_data in data.items(): - logger.info(f"Processing {split_name} split with {len(split_data)} items...") + print(f"Processing {split_name} split with {len(split_data)} items...") processed_data = [] processed_count = 0 @@ -518,10 +514,10 @@ class CustomInstructDataLoader(BaseInstructDataLoader): else: skipped_count += 1 if skipped_count <= 3: # Log first few skipped items - logger.info(f"Skipped item {i} from {split_name}: {item}") + print(f"Skipped item {i} from {split_name}: {item}") processed_splits[split_name] = processed_data - logger.info(f"{split_name} - Preprocessed {processed_count} samples, skipped {skipped_count} samples") + print(f"{split_name} - Preprocessed {processed_count} samples, skipped {skipped_count} samples") return processed_splits @@ -621,59 +617,59 @@ class InstructDataPipeline: encoding=config_dict.get('encoding', 'utf-8') ) - logger.info(f"Configuration loaded from YAML: {yaml_path}") - logger.info(f"Output directory: {config.output_dir}") - logger.info(f"Conversation field: {config.conversation_field}") + print(f"Configuration loaded from YAML: {yaml_path}") + print(f"Output directory: {config.output_dir}") + print(f"Conversation field: {config.conversation_field}") return config except Exception as e: - logger.error(f"Error loading configuration from YAML {yaml_path}: {e}") + print(f"Error loading configuration from YAML {yaml_path}: {e}") raise def load_and_preprocess(self, config: InstructConfig) -> Tuple[Dict[str, List[Dict]], Dict[str, Any]]: """Load and preprocess conversation data""" - logger.info(f"Starting conversation data loading and preprocessing...") - logger.info(f"Data source: {config.data_source}") + print(f"Starting conversation data loading and preprocessing...") + print(f"Data source: {config.data_source}") try: # Load data if config.data_source == "huggingface": - logger.info("Loading HuggingFace conversation dataset...") + print("Loading HuggingFace conversation dataset...") raw_splits = self.hf_loader.load(config) - logger.info("Preprocessing HuggingFace conversation dataset...") + print("Preprocessing HuggingFace conversation dataset...") processed_splits = self.hf_loader.preprocess(raw_splits, config) elif config.data_source == "custom": - logger.info("Loading custom conversation dataset...") + print("Loading custom conversation dataset...") raw_splits = self.custom_loader.load(config) - logger.info("Preprocessing custom conversation dataset...") + print("Preprocessing custom conversation dataset...") processed_splits = self.custom_loader.preprocess(raw_splits, config) else: raise ValueError(f"Unsupported data source: {config.data_source}") - logger.info(f"Conversation data loading and preprocessing completed successfully") + print(f"Conversation data loading and preprocessing completed successfully") # Validate processed data - logger.info("Validating processed conversation data...") + print("Validating processed conversation data...") is_valid, errors = self.validator.validate_conversation_data(processed_splits, config, is_processed=True) if not is_valid: - logger.error("Conversation data validation failed:") + print("Conversation data validation failed:") for error in errors: - logger.error(f" - {error}") + print(f" - {error}") raise ValueError("Conversation data validation failed") - logger.info("Conversation data validation passed") + print("Conversation data validation passed") # Analyze dataset - logger.info("Analyzing conversation dataset...") + print("Analyzing conversation dataset...") analysis = self.validator.analyze_conversation_dataset(processed_splits, config, is_processed=True) - logger.info("Conversation dataset analysis completed") + print("Conversation dataset analysis completed") return processed_splits, analysis except Exception as e: - logger.error(f"Error in load_and_preprocess: {e}") + print(f"Error in load_and_preprocess: {e}") raise def save_data(self, data: Dict[str, List[Dict]], output_dir: str, format: str = "jsonl"): @@ -692,7 +688,7 @@ class InstructDataPipeline: with open(output_file, 'w', encoding='utf-8') as f: json.dump(split_data, f, ensure_ascii=False, indent=2) - logger.info(f"Saved {len(split_data)} conversation samples to {output_file}") + print(f"Saved {len(split_data)} conversation samples to {output_file}") def run_pipeline( self, @@ -701,7 +697,7 @@ class InstructDataPipeline: ) -> Dict[str, Any]: """Run complete instruction data pipeline""" - logger.info("Starting instruction data pipeline...") + print("Starting instruction data pipeline...") # Load and preprocess data processed_splits, analysis = self.load_and_preprocess(config) @@ -723,7 +719,7 @@ class InstructDataPipeline: "data": processed_splits, # Include the actual processed data } - logger.info("Instruction data pipeline completed successfully!") + print("Instruction data pipeline completed successfully!") return result def load_yaml_config(config_path: str) -> Dict[str, Any]: @@ -764,13 +760,13 @@ def load_yaml_config(config_path: str) -> Dict[str, Any]: 'encoding': data_config.get('encoding') }) - logger.info(f"Successfully parsed YAML configuration from: {config_path}") - logger.info(f"Extracted {len(config_dict)} configuration parameters") + print(f"Successfully parsed YAML configuration from: {config_path}") + print(f"Extracted {len(config_dict)} configuration parameters") return config_dict except Exception as e: - logger.error(f"Error loading YAML config from {config_path}: {e}") + print(f"Error loading YAML config from {config_path}: {e}") raise def main(): @@ -805,10 +801,10 @@ def main(): args = parser.parse_args() # Set up logging - logging.basicConfig( - level=getattr(logging, args.log_level), - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' - ) + # logging.basicConfig( + # level=getattr(logging, args.log_level), + # format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + # ) # Load configuration config_dict = {} @@ -818,7 +814,7 @@ def main(): try: config_dict = load_yaml_config(args.config) except Exception as e: - logger.error(f"Error loading YAML config: {e}") + print(f"Error loading YAML config: {e}") sys.exit(1) # Override YAML config with CLI arguments (similar to styling pipeline) @@ -847,7 +843,7 @@ def main(): # Merge configurations for key, value in cli_overrides.items(): if key in config_dict: - logger.info(f"Overriding YAML config '{key}' with CLI value: {value}") + print(f"Overriding YAML config '{key}' with CLI value: {value}") config_dict[key] = value # Validate required arguments diff --git a/pipelines/instruct/train.py b/pipelines/instruct/train.py index aa03480..c37dd0c 100644 --- a/pipelines/instruct/train.py +++ b/pipelines/instruct/train.py @@ -7,7 +7,6 @@ Supports instruction fine-tuning with conversational data and LoRA fine-tuning import os import sys import json -import logging import argparse from pathlib import Path from typing import Dict, Any, Optional, List @@ -21,13 +20,11 @@ from utils.config.config_manager import ConfigManager # Training imports import torch from datasets import load_from_disk, Dataset -from unsloth import FastLanguageModel, is_bfloat16_supported +from unsloth import FastLanguageModel #is_bfloat16_supported from unsloth.chat_templates import get_chat_template, standardize_sharegpt, train_on_responses_only from trl import SFTTrainer, SFTConfig from transformers import DataCollatorForSeq2Seq -logger = logging.getLogger(__name__) - class InstructTrainer: """Instruction fine-tuning trainer using Unsloth and SFTTrainer""" @@ -39,7 +36,7 @@ class InstructTrainer: # Set device self.device = "cuda" if torch.cuda.is_available() else "cpu" - logger.info(f"Using device: {self.device}") + print(f"Using device: {self.device}") # Model parameters self.model_name = config.get('model_name', 'unsloth/Qwen2.5-72B-Instruct') @@ -76,7 +73,7 @@ class InstructTrainer: def load_model_and_tokenizer(self): """Load the pre-trained model and tokenizer""" - logger.info("Loading model and tokenizer...") + print("Loading model and tokenizer...") try: self.model, self.tokenizer = FastLanguageModel.from_pretrained( @@ -87,16 +84,16 @@ class InstructTrainer: token=self.hf_token ) - logger.info(f"✅ Model loaded: {self.model_name}") - logger.info(f"✅ Tokenizer loaded with vocab size: {self.tokenizer.vocab_size}") + print(f"✅ Model loaded: {self.model_name}") + print(f"✅ Tokenizer loaded with vocab size: {self.tokenizer.vocab_size}") except Exception as e: - logger.error(f"❌ Error loading model: {e}") + print(f"❌ Error loading model: {e}") raise def setup_lora(self): """Setup LoRA for efficient fine-tuning""" - logger.info("Setting up LoRA configuration...") + print("Setting up LoRA configuration...") try: self.model = FastLanguageModel.get_peft_model( @@ -112,15 +109,15 @@ class InstructTrainer: loftq_config=None ) - logger.info(f"✅ LoRA configured with r={self.lora_r}, alpha={self.lora_alpha}") + print(f"✅ LoRA configured with r={self.lora_r}, alpha={self.lora_alpha}") except Exception as e: - logger.error(f"❌ Error setting up LoRA: {e}") + print(f"❌ Error setting up LoRA: {e}") raise def setup_chat_template(self): """Setup chat template for conversation formatting""" - logger.info("Setting up chat template...") + print("Setting up chat template...") try: self.tokenizer = get_chat_template( @@ -128,15 +125,15 @@ class InstructTrainer: chat_template=self.chat_template, ) - logger.info(f"✅ Chat template configured: {self.chat_template}") + print(f"✅ Chat template configured: {self.chat_template}") except Exception as e: - logger.error(f"❌ Error setting up chat template: {e}") + print(f"❌ Error setting up chat template: {e}") raise def load_dataset(self, dataset_path: str) -> Dataset: """Load the conversation training dataset""" - logger.info(f"Loading conversation dataset from: {dataset_path}") + print(f"Loading conversation dataset from: {dataset_path}") try: if Path(dataset_path).exists(): @@ -144,10 +141,10 @@ class InstructTrainer: if (Path(dataset_path) / "dataset_info.json").exists(): # Load from HuggingFace dataset directory dataset = load_from_disk(dataset_path) - logger.info(f"Loaded HuggingFace dataset from disk: {len(dataset)} samples") + print(f"Loaded HuggingFace dataset from disk: {len(dataset)} samples") else: # Load from processed conversation data files (JSONL format) - logger.info("Loading from processed conversation data files...") + print("Loading from processed conversation data files...") from datasets import Dataset import json @@ -158,7 +155,7 @@ class InstructTrainer: for split_file in ["train.jsonl", "validation.jsonl", "test.jsonl"]: file_path = data_dir / split_file if file_path.exists(): - logger.info(f"Loading {split_file}...") + print(f"Loading {split_file}...") with open(file_path, 'r', encoding='utf-8') as f: for line in f: if line.strip(): @@ -170,15 +167,15 @@ class InstructTrainer: # Create HuggingFace dataset dataset = Dataset.from_list(all_data) - logger.info(f"Created HuggingFace dataset from {len(all_data)} conversation samples") + print(f"Created HuggingFace dataset from {len(all_data)} conversation samples") else: # Try loading from HuggingFace Hub - logger.info(f"Attempting to load from HuggingFace Hub: {dataset_path}") + print(f"Attempting to load from HuggingFace Hub: {dataset_path}") dataset = Dataset.load_dataset(dataset_path, split="train") - logger.info(f"Loaded from HuggingFace Hub: {len(dataset)} samples") + print(f"Loaded from HuggingFace Hub: {len(dataset)} samples") - logger.info(f"Dataset loaded: {len(dataset)} samples") - logger.info(f"Dataset features: {dataset.features}") + print(f"Dataset loaded: {len(dataset)} samples") + print(f"Dataset features: {dataset.features}") # Verify required fields exist for conversation data required_fields = ["conversation"] @@ -189,16 +186,16 @@ class InstructTrainer: return dataset except Exception as e: - logger.error(f"Error loading conversation dataset: {e}") + print(f"Error loading conversation dataset: {e}") raise def format_dataset_for_training(self, dataset: Dataset) -> Dataset: """Format conversation dataset for training using standardize_sharegpt and apply_chat_template""" - logger.info("Formatting conversation dataset for training...") + print("Formatting conversation dataset for training...") try: # Standardize the ShareGPT format - logger.info("Standardizing ShareGPT format...") + print("Standardizing ShareGPT format...") dataset = standardize_sharegpt(dataset) # Define the formatting function for chat templates @@ -214,21 +211,21 @@ class InstructTrainer: return {"text": texts} # Apply the formatting function - logger.info("Applying chat template formatting...") + print("Applying chat template formatting...") dataset = dataset.map(formatting_prompts_func, batched=True) - logger.info(f"✅ Dataset formatted for training with {len(dataset)} samples") - logger.info(f"Sample formatted text: {dataset[0]['text'][:200]}...") + print(f"✅ Dataset formatted for training with {len(dataset)} samples") + print(f"Sample formatted text: {dataset[0]['text'][:200]}...") return dataset except Exception as e: - logger.error(f"❌ Error formatting dataset: {e}") + print(f"❌ Error formatting dataset: {e}") raise def setup_trainer(self, train_dataset: Dataset): """Setup the SFTTrainer for instruction fine-tuning""" - logger.info("Setting up SFTTrainer for instruction fine-tuning...") + print("Setting up SFTTrainer for instruction fine-tuning...") try: # SFT Configuration @@ -247,12 +244,12 @@ class InstructTrainer: report_to="none", # Disable wandb for now ) - logger.info("SFT Configuration:") - logger.info(f" batch_size: {self.batch_size}") - logger.info(f" gradient_accumulation_steps: {self.gradient_accumulation_steps}") - logger.info(f" warmup_steps: {self.warmup_steps}") - logger.info(f" max_steps: {self.max_steps}") - logger.info(f" learning_rate: {self.learning_rate}") + print("SFT Configuration:") + print(f" batch_size: {self.batch_size}") + print(f" gradient_accumulation_steps: {self.gradient_accumulation_steps}") + print(f" warmup_steps: {self.warmup_steps}") + print(f" max_steps: {self.max_steps}") + print(f" learning_rate: {self.learning_rate}") # Create SFTTrainer self.trainer = SFTTrainer( @@ -266,18 +263,18 @@ class InstructTrainer: args=sft_config, ) - logger.info("✅ SFTTrainer configured successfully") + print("✅ SFTTrainer configured successfully") except Exception as e: - logger.error(f"❌ Error setting up trainer: {e}") + print(f"❌ Error setting up trainer: {e}") import traceback - logger.error("Full error traceback:") + print("Full error traceback:") traceback.print_exc() raise def setup_response_only_training(self): """Setup training to only learn from assistant responses""" - logger.info("Setting up response-only training...") + print("Setting up response-only training...") try: # Configure trainer to only train on responses @@ -287,51 +284,51 @@ class InstructTrainer: response_part="<|im_start|>assistant\n", ) - logger.info("✅ Response-only training configured") + print("✅ Response-only training configured") except Exception as e: - logger.error(f"❌ Error setting up response-only training: {e}") + print(f"❌ Error setting up response-only training: {e}") raise def train(self, dataset_path: str): """Run the instruction fine-tuning process""" - logger.info("🚀 Starting instruction fine-tuning process...") + print("🚀 Starting instruction fine-tuning process...") try: # Load model and tokenizer - logger.info("Step 1: Loading model and tokenizer...") + print("Step 1: Loading model and tokenizer...") self.load_model_and_tokenizer() # Setup LoRA - logger.info("Step 2: Setting up LoRA...") + print("Step 2: Setting up LoRA...") self.setup_lora() # Setup chat template - logger.info("Step 3: Setting up chat template...") + print("Step 3: Setting up chat template...") self.setup_chat_template() # Load dataset - logger.info(f"Step 4: Loading conversation dataset from: {dataset_path}") + print(f"Step 4: Loading conversation dataset from: {dataset_path}") train_dataset = self.load_dataset(dataset_path) # Format dataset for training - logger.info("Step 5: Formatting dataset for training...") + print("Step 5: Formatting dataset for training...") formatted_dataset = self.format_dataset_for_training(train_dataset) # Setup trainer - logger.info("Step 6: Setting up trainer...") + print("Step 6: Setting up trainer...") self.setup_trainer(formatted_dataset) # Setup response-only training (optional but recommended for chat models) - logger.info("Step 7: Setting up response-only training...") + print("Step 7: Setting up response-only training...") self.setup_response_only_training() # Start training - logger.info("Step 8: Starting training...") + print("Step 8: Starting training...") trainer_stats = self.trainer.train() - logger.info("✅ Instruction fine-tuning completed successfully!") - logger.info(f"Training stats: {trainer_stats}") + print("✅ Instruction fine-tuning completed successfully!") + print(f"Training stats: {trainer_stats}") # Save the model self.save_model() @@ -339,15 +336,15 @@ class InstructTrainer: return trainer_stats except Exception as e: - logger.error(f"❌ Instruction fine-tuning failed: {e}") + print(f"❌ Instruction fine-tuning failed: {e}") import traceback - logger.error("Full error traceback:") + print("Full error traceback:") traceback.print_exc() raise def save_model(self): """Save the trained instruction model""" - logger.info("Saving trained instruction model...") + print("Saving trained instruction model...") try: # Create output directory @@ -362,23 +359,23 @@ class InstructTrainer: with open(config_path, 'w') as f: json.dump(self.config, f, indent=2) - logger.info(f"✅ Instruction model saved to: {self.model_output_dir}") - logger.info(f"✅ You can now use this model for inference") + print(f"✅ Instruction model saved to: {self.model_output_dir}") + print(f"✅ You can now use this model for inference") except Exception as e: - logger.error(f"❌ Error saving model: {e}") + print(f"❌ Error saving model: {e}") raise def prepare_for_inference(self): """Prepare model for inference""" - logger.info("Preparing model for inference...") + print("Preparing model for inference...") try: FastLanguageModel.for_inference(self.model) - logger.info("✅ Model prepared for inference") + print("✅ Model prepared for inference") except Exception as e: - logger.error(f"❌ Error preparing for inference: {e}") + print(f"❌ Error preparing for inference: {e}") raise def load_training_config(yaml_path: str) -> Dict[str, Any]: @@ -403,16 +400,16 @@ def load_training_config(yaml_path: str) -> Dict[str, Any]: # Training configuration - extract from training section if 'training' in config: training_data = config['training'] - logger.info("Training data from YAML:") - logger.info(f" num_epochs: {training_data.get('num_epochs')} (type: {type(training_data.get('num_epochs'))})") - logger.info(f" batch_size: {training_data.get('batch_size')} (type: {type(training_data.get('batch_size'))})") - logger.info(f" learning_rate: {training_data.get('learning_rate')} (type: {type(training_data.get('learning_rate'))})") - logger.info(f" weight_decay: {training_data.get('weight_decay')} (type: {type(training_data.get('weight_decay'))})") - logger.info(f" warmup_steps: {training_data.get('warmup_steps')} (type: {type(training_data.get('warmup_steps'))})") - logger.info(f" max_steps: {training_data.get('max_steps')} (type: {type(training_data.get('max_steps'))})") - logger.info(f" gradient_accumulation_steps: {training_data.get('gradient_accumulation_steps')} (type: {type(training_data.get('gradient_accumulation_steps'))})") - logger.info(f" seed: {training_data.get('seed')} (type: {type(training_data.get('seed'))})") - logger.info(f" model_output_dir: {training_data.get('model_output_dir')} (type: {type(training_data.get('model_output_dir'))})") + print("Training data from YAML:") + print(f" num_epochs: {training_data.get('num_epochs')} (type: {type(training_data.get('num_epochs'))})") + print(f" batch_size: {training_data.get('batch_size')} (type: {type(training_data.get('batch_size'))})") + print(f" learning_rate: {training_data.get('learning_rate')} (type: {type(training_data.get('learning_rate'))})") + print(f" weight_decay: {training_data.get('weight_decay')} (type: {type(training_data.get('weight_decay'))})") + print(f" warmup_steps: {training_data.get('warmup_steps')} (type: {type(training_data.get('warmup_steps'))})") + print(f" max_steps: {training_data.get('max_steps')} (type: {type(training_data.get('max_steps'))})") + print(f" gradient_accumulation_steps: {training_data.get('gradient_accumulation_steps')} (type: {type(training_data.get('gradient_accumulation_steps'))})") + print(f" seed: {training_data.get('seed')} (type: {type(training_data.get('seed'))})") + print(f" model_output_dir: {training_data.get('model_output_dir')} (type: {type(training_data.get('model_output_dir'))})") training_config.update({ 'num_epochs': int(training_data.get('num_epochs', 1)), @@ -450,14 +447,14 @@ def load_training_config(yaml_path: str) -> Dict[str, Any]: 'chat_template': 'qwen-2.5' # Use Qwen chat template by default }) - logger.info("Final training_config:") + print("Final training_config:") for key, value in training_config.items(): - logger.info(f" {key}: {value} (type: {type(value)})") + print(f" {key}: {value} (type: {type(value)})") return training_config except Exception as e: - logger.error(f"Error loading training config: {e}") + print(f"Error loading training config: {e}") raise def main(): @@ -475,15 +472,11 @@ def main(): args = parser.parse_args() - # Setup logging - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' - ) + # Setup logging replaced with print statements try: # Load configuration - logger.info(f"Loading configuration from: {args.config}") + print(f"Loading configuration from: {args.config}") training_config = load_training_config(args.config) # Override with CLI arguments @@ -501,13 +494,13 @@ def main(): # Determine dataset path: CLI argument takes precedence, then YAML config dataset_path = args.dataset or training_config.get('dataset_path') if not dataset_path: - logger.error("No dataset path provided. Use --dataset or ensure output_dir is set in YAML config.") + print("No dataset path provided. Use --dataset or ensure output_dir is set in YAML config.") sys.exit(1) - logger.info("Training configuration:") + print("Training configuration:") for key, value in training_config.items(): - logger.info(f" {key}: {value}") - logger.info(f" Dataset path: {dataset_path}") + print(f" {key}: {value}") + print(f" Dataset path: {dataset_path}") # Initialize trainer trainer = InstructTrainer(training_config) @@ -515,10 +508,10 @@ def main(): # Start training trainer.train(dataset_path) - logger.info("Instruction fine-tuning completed successfully!") + print("Instruction fine-tuning completed successfully!") except Exception as e: - logger.error(f"Instruction fine-tuning failed: {e}") + print(f"Instruction fine-tuning failed: {e}") sys.exit(1) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 862bfea..5346e99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ numpy>=1.24.0 scikit-learn>=1.3.0 pyyaml>=6.0 huggingface-hub>=0.15.0 +unsloth diff --git a/setup.sh b/setup.sh new file mode 100644 index 0000000..51b0baa --- /dev/null +++ b/setup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -e # exit on error + +# Step 1: Download Miniconda installer +echo ">>> Downloading Miniconda..." +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh + +# Step 2: Run the installer +echo ">>> Installing Miniconda..." +bash ~/miniconda.sh -b -p $HOME/miniconda + +# Step 3: Initialize conda for bash +echo ">>> Initializing Conda..." +eval "$($HOME/miniconda/bin/conda shell.bash hook)" + +# Step 4: Add conda to PATH permanently +if ! grep -q "miniconda/bin" ~/.bashrc; then + echo "export PATH=\$HOME/miniconda/bin:\$PATH" >> ~/.bashrc +fi +source ~/.bashrc + +# Step 5: Verify installation +echo ">>> Conda version:" +conda --version + +# Step 6: Create environment +echo ">>> Creating environment: llm-env (python=3.10)..." +conda create -n llm-env python=3.10 -y + +# Step 7: Activate environment +echo ">>> Activating environment..." +conda activate llm-env + +# Step 8: Install dependencies +echo ">>> Installing scikit-learn..." +conda install scikit-learn -y + +if [ -f requirements.txt ]; then + echo ">>> Installing Python requirements..." + pip install -r requirements.txt +else + echo ">>> No requirements.txt found, skipping pip install." +fi + +echo ">>> Setup complete. To activate your environment run:" +echo "conda activate llm-env" diff --git a/utils/__pycache__/__init__.cpython-310.pyc b/utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd3da5adee1cd8133a696029e606c0b2cf40d5e2 GIT binary patch literal 148 zcmd1j<>g`kf9AJhN< literal 0 HcmV?d00001 diff --git a/utils/config/__pycache__/__init__.cpython-310.pyc b/utils/config/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f572770a96415523b938c7d9463d18fee0f2a2f0 GIT binary patch literal 155 zcmd1j<>g`kfBn*Hs$RLt|w2&5_l&MzwxT zj;W#Dx4C_|#Op_Dzj0e(1~Z>3%zUEto6O=ZR^ujbaciKhDFcJqZ0V`dUuHB{wD;=^g3!`-1E}Cj$RmBfu9!o_2_Y7-yNqx9C@KnOx52Kh!623 z|3SkQ+D2bvYF}p>)1NARgBjdpCbytiZK{^VYEa8&_Ons8ROwf_Ek;Tj{Uz3{aI_$E z8S*T)!rGXvvvbeQpX4>HTN~Ky6kEkM8|*wgEp=%=Yyq4z>?}r?*{jfSg_{6p32$qr zC+K-S*4R4u+w26u+9+B#;%E>Icf80Oa*=;rb`r;z!+T{nn=)ODM*Jn2za^V4(t-kh-%jZ%y|hVK-X;AtizpT3#>Omu8K zaxV|jNtKEEL_N?Z+M#-+Br7T0W}<$p{9SvZ{z*;siAFBOjBoV6whr{vI5a2vk$PZ! z2E!PNp*+DkiR^ckAP5SJe;p)gQk=L2p}jc0 zMIvts5sSiz$2=;G$KEI`mJ)BkUGka23gaQnP#8fph&!gBV}R1vkF@N2=uFf@oY*A9#Wb7C?zQ^lEya;Q!8i7_#cV8ciDM2C&E z{JkA6hP*r`rP?!Fd=;d7FF7wg3RHd!cQu2QkASh8@YIcXl5*zmdtt^qRteuaR1@dW z6s-}h#HQdB`j;G^OWs8!OU@}{avmQibQ)}}t=sCds{PT>@^hu*%pjDqXJ3@&aqx+F z4UKQ0nf)jm5G`^07#-k+oIpN4QGu30{{Lr0av->TU`?!vxexRZ1I&qaRkmx&f%&Kf zEQwoRQ?4S1081m^+2yI@IVB8^+?o?a(*xjS2?Zqv=Z1rqk?Jzv9B#htTuOz9Lmv${ zA9$(1=O#hUFLerav-kav_`&tfWE=)*r&eMrz0ZN}a(jgVB$J}%XF?#D73!A@c!~|M zUc!?sqp52KT+vo-wT^dwVIIfl?AR-6j1RmlOkWP;HHck=5%8Tluu^qmrrHxttVtjh zyaA&GaA~4z2pzkCR663jx1R@328+@wUjUzY1M7Bl@fKc1JpjB(>P0>;42B%~JXjs@ ziHp<_7~(QD6uGNscnCKx7cAZQXP7^R&DKr=G+TrFZOp^A_}LH6yNhfOKSYF1^)f~y(rTbhhU7lfsPgy* zXIqZd!PB5PKBb$WTNQeFsIZlPsH|Q2<^*_7R(MW8JTGZV2wB z&oaC-_C&&klkRz`lVsy@EYhSJ^vV*Uql!%+pWEGTX{`JLm1RyES9xavl9EA3D~$B< z7(Tbh!*Sl~cA1xY-6YP0&-;zNY&gWZ9-t=ZTYi?L@hER>fx{^|@OjPOivyo0;vHIf zxfFnC+Ks#s&(D5bO1K3H4vEH$BWD`Zr6~93XJ=}bYe-JHrDKf-u^4%2NAH^tlQ;_d zx}WS92KA>3l?=omMB^;&4gwysynTlVhyqcr3x?2pz{45CHW3R9Bk${O;*G|ry7KiM z@9SWcjhyKKfjJg2cAVtRZa2i28)C| z8e;orj@RIiGE@oNnNEN$%0^v2j{UtPZ}+m%19(9^aHb=emm#-HAvdpoMB&$w{S|DJ zdxD~J7j-4iTia98k>s&z68}>({1a~^T+NO8)0kx{G>IjsV=L6--1ZmNro*+TXNCCos<)Q{O8k@TMD0f-@L}s3+E2lY!8GAA`|VQ^ffBY7q!1IjUJ2o zyL2vaOZo#TSni{#ir