video preprocessing pipeline completed
This commit is contained in:
timothyafolami
File diff suppressed because one or more lines are too long
@@ -134,3 +134,10 @@
|
||||
2024-08-09 16:32:44,664 - INFO - Search completed
|
||||
2024-08-09 16:32:44,665 - INFO - Page content: Oilchange pouring new oil in the engine
|
||||
|
||||
2024-08-09 16:52:52,869 - INFO - Receiving the search query
|
||||
2024-08-09 16:53:04,624 - INFO - Searching for Anti-squeal paste
|
||||
2024-08-09 16:53:05,451 - INFO - Search completed
|
||||
2024-08-09 16:54:31,010 - INFO - Receiving the search query
|
||||
2024-08-09 16:54:37,491 - INFO - Searching for Welcome back to Toyota Maintenance YouTube channel
|
||||
2024-08-09 16:54:38,251 - INFO - Search completed
|
||||
2024-08-09 16:54:38,251 - INFO - Page content: this without any hesitation buy one of these to just get to the work and back and save a lot of money on the gas that's my feeling you please share underneath the video your experience with others because that's why we filming this creating community of Toyota enthusiasts who share their own own experience if you like the video give it a thumb up and be subscribed i will always have a lot of new stuff coming your way thank you for watching and have a great day my friend
|
||||
|
||||
+144
-27
@@ -2,52 +2,155 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"source": [
|
||||
"# !pip install moviepy\n",
|
||||
"# !pip install ffmpeg-python"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"source": [
|
||||
"from moviepy.editor import VideoFileClip\n",
|
||||
"import os\n",
|
||||
"import ffmpeg\n",
|
||||
"# importing module that prerocess the audio file \n",
|
||||
"from data_ingestion.utils import create_audio_document\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Video Preprocessing Pipeline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"source": [
|
||||
"video_path = 'data/How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC].mp4'\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# creating a function to preprocess the videos\n",
|
||||
"def preprocess_video_data(video_path: str, time_interval: int):\n",
|
||||
" \n",
|
||||
" # Load the video file\n",
|
||||
" video = VideoFileClip(video_path)\n",
|
||||
" \n",
|
||||
" # Get the duration of the video\n",
|
||||
" duration = video.duration\n",
|
||||
" \n",
|
||||
" # create an audio version of the video\n",
|
||||
" audio_path = video_path.replace('.mp4', '.mp3')\n",
|
||||
" _ = video.audio.write_audiofile(audio_path)\n",
|
||||
" \n",
|
||||
" # creating a snapshot of the videos at the time interval\n",
|
||||
" # Extract the video filename without extension\n",
|
||||
" video_name = os.path.splitext(os.path.basename(video_path))[0]\n",
|
||||
"\n",
|
||||
" # Create a directory for snapshots using the video name\n",
|
||||
" snapshot_dir = os.path.join(os.path.dirname(video_path), f\"{video_name}_snapshots\")\n",
|
||||
" os.makedirs(snapshot_dir, exist_ok=True)\n",
|
||||
"\n",
|
||||
" # Set the interval to 3 minutes (180 seconds)\n",
|
||||
" interval = 180\n",
|
||||
"\n",
|
||||
" # Get the duration of the video using ffmpeg\n",
|
||||
" probe = ffmpeg.probe(video_path)\n",
|
||||
" duration = float(probe['format']['duration'])\n",
|
||||
"\n",
|
||||
" # Loop through the video and take snapshots at 0s, 3min, 6min, etc.\n",
|
||||
" for i in range(0, int(duration), interval):\n",
|
||||
" # Calculate the time for the current frame\n",
|
||||
" frame_time = i\n",
|
||||
" # Save the snapshot as an image file in the created folder\n",
|
||||
" frame_img = os.path.join(snapshot_dir, f\"frame_at_{frame_time//60}min.png\")\n",
|
||||
" \n",
|
||||
" # Extract the frame using ffmpeg\n",
|
||||
" (\n",
|
||||
" ffmpeg\n",
|
||||
" .input(video_path, ss=frame_time)\n",
|
||||
" .output(frame_img, vframes=1)\n",
|
||||
" .run()\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" print(f\"Snapshots saved in {snapshot_dir}.\")\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" # now creating document from the audio file\n",
|
||||
" documents = create_audio_document(audio_path)\n",
|
||||
" return documents\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"MoviePy - Writing audio in data/How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC].mp3\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" \r"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"MoviePy - Done.\n",
|
||||
"Snapshots saved in data\\How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_snapshots.\n",
|
||||
"Exporting How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunks/How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunk1.mp3\n",
|
||||
"Exporting How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunks/How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunk2.mp3\n",
|
||||
"Exporting How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunks/How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunk3.mp3\n",
|
||||
"Exporting How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunks/How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]_chunk4.mp3\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Implementing the function\n",
|
||||
"documents = preprocess_video_data(video_path, 180)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[Document(metadata={'filename': 'How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]', 'duration': '0-3 minutes'}, page_content=\" Hi everybody! Here's the latest installment of AutoDoc's video tutorials on replacing car parts. The channel so you never miss a video. We post new ones every week!\"),\n",
|
||||
" Document(metadata={'filename': 'How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]', 'duration': '3-6 minutes'}, page_content=\" How to make a You can buy spare parts from us on our website or in the Autodoc app. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. 15. Remove the rear window cover. 16. Remove the rear window cover. 17. Remove the rear window cover. 18. Remove the rear window cover. 19. Remove the rear window cover. I'll see you next time.\"),\n",
|
||||
" Document(metadata={'filename': 'How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]', 'duration': '6-9 minutes'}, page_content=\" How to make a Are you interested in this product? All links can be found in the description. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. I'm going to make a hole in the bottom of the box. Add the so I going to make a fire with a fire extinguisher I going to make a fire with a fire extinguisher I'm going to make a fire with a fire extinguisher. I'm going to make a fire with a fire extinguisher. I'm going to make a fire with a fire extinguisher. The\"),\n",
|
||||
" Document(metadata={'filename': 'How to change front wheel bearing on TOYOTA RAV4 II [TUTORIAL AUTODOC]', 'duration': '9-12 minutes'}, page_content=\" I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. I'm going to make a new one. Thank you for watching our video tutorials. If you enjoyed watching, click thumbs up and share it with your friends. Have a nice day! Follow us on social media. Find us on Instagram and TikTok!\")]"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"documents"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -93,8 +196,22 @@
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "smog_env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Reference in New Issue
Block a user