This commit is contained in:
Eric Ciarla
2024-07-23 15:48:12 -04:00
parent 252bc09ee2
commit a0d89169ed
24 changed files with 5893 additions and 0 deletions
@@ -0,0 +1,68 @@
import { NextResponse } from 'next/server';
const FIRECRAWL_API_URL = process.env.FIRECRAWL_API_URL;
const FIRECRAWL_API_KEY = process.env.FIRECRAWL_API_KEY;
export async function POST(request: Request) {
try {
const body = await request.json();
console.log(body);
const {
url,
crawlSubPages,
limit,
maxDepth,
excludePaths,
includePaths,
extractMainContent
} = body;
const endpoint = `${FIRECRAWL_API_URL}/v0/${crawlSubPages ? 'crawl' : 'scrape'}`;
const requestBody = crawlSubPages ? {
url,
crawlerOptions: {
includes: includePaths ? includePaths.split(',').map((p: string) => p.trim()) : undefined,
excludes: excludePaths ? excludePaths.split(',').map((p: string) => p.trim()) : undefined,
maxDepth: maxDepth ? parseInt(maxDepth) : undefined,
limit: limit ? parseInt(limit) : undefined,
},
pageOptions: {
onlyMainContent: extractMainContent,
}
} : {
url,
pageOptions: {
onlyMainContent: extractMainContent,
}
};
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'Authorization': `Bearer ${FIRECRAWL_API_KEY}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
});
if (!response.ok) {
throw new Error(`Firecrawl API responded with status ${response.status}`);
}
const firecrawlResponse = await response.json();
return NextResponse.json({
success: true,
message: crawlSubPages ? 'Crawl process started' : 'Scrape process completed',
data: firecrawlResponse,
});
} catch (error) {
console.error('Error processing ingestion request:', error);
return NextResponse.json(
{ success: false, message: 'Error processing ingestion request' },
{ status: 500 }
);
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

+69
View File
@@ -0,0 +1,69 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
@layer base {
:root {
--background: 0 0% 100%;
--foreground: 222.2 84% 4.9%;
--card: 0 0% 100%;
--card-foreground: 222.2 84% 4.9%;
--popover: 0 0% 100%;
--popover-foreground: 222.2 84% 4.9%;
--primary: 222.2 47.4% 11.2%;
--primary-foreground: 210 40% 98%;
--secondary: 210 40% 96.1%;
--secondary-foreground: 222.2 47.4% 11.2%;
--muted: 210 40% 96.1%;
--muted-foreground: 215.4 16.3% 46.9%;
--accent: 210 40% 96.1%;
--accent-foreground: 222.2 47.4% 11.2%;
--destructive: 0 84.2% 60.2%;
--destructive-foreground: 210 40% 98%;
--border: 214.3 31.8% 91.4%;
--input: 214.3 31.8% 91.4%;
--ring: 222.2 84% 4.9%;
--radius: 0.5rem;
--chart-1: 12 76% 61%;
--chart-2: 173 58% 39%;
--chart-3: 197 37% 24%;
--chart-4: 43 74% 66%;
--chart-5: 27 87% 67%;
}
.dark {
--background: 222.2 84% 4.9%;
--foreground: 210 40% 98%;
--card: 222.2 84% 4.9%;
--card-foreground: 210 40% 98%;
--popover: 222.2 84% 4.9%;
--popover-foreground: 210 40% 98%;
--primary: 210 40% 98%;
--primary-foreground: 222.2 47.4% 11.2%;
--secondary: 217.2 32.6% 17.5%;
--secondary-foreground: 210 40% 98%;
--muted: 217.2 32.6% 17.5%;
--muted-foreground: 215 20.2% 65.1%;
--accent: 217.2 32.6% 17.5%;
--accent-foreground: 210 40% 98%;
--destructive: 0 62.8% 30.6%;
--destructive-foreground: 210 40% 98%;
--border: 217.2 32.6% 17.5%;
--input: 217.2 32.6% 17.5%;
--ring: 212.7 26.8% 83.9%;
--chart-1: 220 70% 50%;
--chart-2: 160 60% 45%;
--chart-3: 30 80% 55%;
--chart-4: 280 65% 60%;
--chart-5: 340 75% 55%;
}
}
@layer base {
* {
@apply border-border;
}
body {
@apply bg-background text-foreground;
}
}
+22
View File
@@ -0,0 +1,22 @@
import type { Metadata } from "next";
import { Inter } from "next/font/google";
import "./globals.css";
const inter = Inter({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "Create Next App",
description: "Generated by create next app",
};
export default function RootLayout({
children,
}: Readonly<{
children: React.ReactNode;
}>) {
return (
<html lang="en">
<body className={inter.className}>{children}</body>
</html>
);
}
+13
View File
@@ -0,0 +1,13 @@
"use client";
import Image from "next/image";
import StartIngestion from "@/components/startIngestion";
export default function Home() {
return (
<main className="flex min-h-screen flex-col items-center justify-between p-24">
<div className="z-10 w-full max-w-5xl items-center justify-between font-mono text-sm lg:flex">
<StartIngestion />
</div>
</main>
);
}
@@ -0,0 +1,198 @@
/**
* v0 by Vercel.
* @see https://v0.dev/t/MHqslFy8CCr
* Documentation: https://v0.dev/docs#integrating-generated-code-into-your-nextjs-app
*/
import { Button } from "@/components/ui/button";
import {
Card,
CardHeader,
CardTitle,
CardContent,
CardFooter,
} from "@/components/ui/card";
import Link from "next/link";
import { Input } from "@/components/ui/input";
import { Checkbox } from "@/components/ui/checkbox";
import { Label } from "@/components/ui/label";
import { useState } from "react";
import { JSX, SVGProps } from "react";
export default function StartIngestion() {
const [url, setUrl] = useState("");
const [crawlSubPages, setCrawlSubPages] = useState(true);
const [limit, setLimit] = useState("10");
const [maxDepth, setMaxDepth] = useState("5");
const [excludePaths, setExcludePaths] = useState("");
const [includePaths, setIncludePaths] = useState("");
const [extractMainContent, setExtractMainContent] = useState(true);
const handleSubmit = async () => {
const body = {
url,
crawlSubPages,
limit: parseInt(limit),
maxDepth: parseInt(maxDepth),
excludePaths,
includePaths,
extractMainContent,
};
try {
const response = await fetch("/api/ingestion", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const contentType = response.headers.get("content-type");
if (!contentType || !contentType.includes("application/json")) {
throw new TypeError("Oops, we haven't got JSON!");
}
const data = await response.json();
console.log(data);
if (data.success) {
console.log("Ingestion started:", data);
// Handle successful response (e.g., show a success message, redirect, etc.)
} else {
console.error("Ingestion failed:", data.message);
// Handle error (e.g., show error message to user)
}
} catch (error) {
console.error("Error submitting ingestion request:", error);
// Handle error (e.g., show error message to user)
}
console.log(body);
};
return (
<div className="max-w-2xl mx-auto p-4">
<Card>
<CardHeader className="flex items-center justify-between">
<CardTitle className="flex items-center space-x-2">
<span>Extract web content with Firecrawl 🔥</span>
</CardTitle>
<Link
href="https://docs.firecrawl.dev/introduction"
className="text-sm text-blue-500"
prefetch={false}
>
Firecrawl docs
</Link>
</CardHeader>
<CardContent className="space-y-4">
<Input
placeholder="https://docs.dify.ai"
className="w-full"
value={url}
onChange={(e) => setUrl(e.target.value)}
/>
<div className="flex items-center space-x-2">
<Button variant="default" onClick={handleSubmit}>
Run
</Button>
<Button variant="ghost" className="ml-auto">
<SettingsIcon className="h-5 w-5" />
</Button>
</div>
<div className="space-y-4">
<div className="flex items-center space-x-2">
<Checkbox
id="crawl-sub-pages"
checked={crawlSubPages}
onCheckedChange={(checked) =>
setCrawlSubPages(checked as boolean)
}
/>
<label htmlFor="crawl-sub-pages" className="text-sm">
Crawl sub-pages
</label>
</div>
<div className="grid grid-cols-2 gap-4">
<div>
<Label htmlFor="limit">Limit *</Label>
<Input
id="limit"
placeholder="10"
value={limit}
onChange={(e) => setLimit(e.target.value)}
/>
</div>
<div>
<Label htmlFor="max-depth">Max depth</Label>
<Input
id="max-depth"
placeholder="5"
value={maxDepth}
onChange={(e) => setMaxDepth(e.target.value)}
/>
</div>
</div>
<div className="grid grid-cols-2 gap-4">
<div>
<Label htmlFor="exclude-paths">Exclude paths</Label>
<Input
id="exclude-paths"
placeholder="blog/, /about/"
value={excludePaths}
onChange={(e) => setExcludePaths(e.target.value)}
/>
</div>
<div>
<Label htmlFor="include-paths">Include only paths</Label>
<Input
id="include-paths"
placeholder="articles/"
value={includePaths}
onChange={(e) => setIncludePaths(e.target.value)}
/>
</div>
</div>
<div className="flex items-center space-x-2">
<Checkbox
id="extract-main-content"
checked={extractMainContent}
onCheckedChange={(checked) =>
setExtractMainContent(checked as boolean)
}
/>
<label htmlFor="extract-main-content" className="text-sm">
Extract only main content (no headers, navs, footers, etc.)
</label>
</div>
</div>
</CardContent>
<CardFooter>
<Button disabled variant="default">
Next
</Button>
</CardFooter>
</Card>
</div>
);
}
function SettingsIcon(
props: JSX.IntrinsicAttributes & SVGProps<SVGSVGElement>
) {
return (
<svg
{...props}
xmlns="http://www.w3.org/2000/svg"
width="24"
height="24"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
>
<path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z" />
<circle cx="12" cy="12" r="3" />
</svg>
);
}
@@ -0,0 +1,56 @@
import * as React from "react"
import { Slot } from "@radix-ui/react-slot"
import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const buttonVariants = cva(
"inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
{
variants: {
variant: {
default: "bg-primary text-primary-foreground hover:bg-primary/90",
destructive:
"bg-destructive text-destructive-foreground hover:bg-destructive/90",
outline:
"border border-input bg-background hover:bg-accent hover:text-accent-foreground",
secondary:
"bg-secondary text-secondary-foreground hover:bg-secondary/80",
ghost: "hover:bg-accent hover:text-accent-foreground",
link: "text-primary underline-offset-4 hover:underline",
},
size: {
default: "h-10 px-4 py-2",
sm: "h-9 rounded-md px-3",
lg: "h-11 rounded-md px-8",
icon: "h-10 w-10",
},
},
defaultVariants: {
variant: "default",
size: "default",
},
}
)
export interface ButtonProps
extends React.ButtonHTMLAttributes<HTMLButtonElement>,
VariantProps<typeof buttonVariants> {
asChild?: boolean
}
const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
({ className, variant, size, asChild = false, ...props }, ref) => {
const Comp = asChild ? Slot : "button"
return (
<Comp
className={cn(buttonVariants({ variant, size, className }))}
ref={ref}
{...props}
/>
)
}
)
Button.displayName = "Button"
export { Button, buttonVariants }
@@ -0,0 +1,79 @@
import * as React from "react"
import { cn } from "@/lib/utils"
const Card = React.forwardRef<
HTMLDivElement,
React.HTMLAttributes<HTMLDivElement>
>(({ className, ...props }, ref) => (
<div
ref={ref}
className={cn(
"rounded-lg border bg-card text-card-foreground shadow-sm",
className
)}
{...props}
/>
))
Card.displayName = "Card"
const CardHeader = React.forwardRef<
HTMLDivElement,
React.HTMLAttributes<HTMLDivElement>
>(({ className, ...props }, ref) => (
<div
ref={ref}
className={cn("flex flex-col space-y-1.5 p-6", className)}
{...props}
/>
))
CardHeader.displayName = "CardHeader"
const CardTitle = React.forwardRef<
HTMLParagraphElement,
React.HTMLAttributes<HTMLHeadingElement>
>(({ className, ...props }, ref) => (
<h3
ref={ref}
className={cn(
"text-2xl font-semibold leading-none tracking-tight",
className
)}
{...props}
/>
))
CardTitle.displayName = "CardTitle"
const CardDescription = React.forwardRef<
HTMLParagraphElement,
React.HTMLAttributes<HTMLParagraphElement>
>(({ className, ...props }, ref) => (
<p
ref={ref}
className={cn("text-sm text-muted-foreground", className)}
{...props}
/>
))
CardDescription.displayName = "CardDescription"
const CardContent = React.forwardRef<
HTMLDivElement,
React.HTMLAttributes<HTMLDivElement>
>(({ className, ...props }, ref) => (
<div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
))
CardContent.displayName = "CardContent"
const CardFooter = React.forwardRef<
HTMLDivElement,
React.HTMLAttributes<HTMLDivElement>
>(({ className, ...props }, ref) => (
<div
ref={ref}
className={cn("flex items-center p-6 pt-0", className)}
{...props}
/>
))
CardFooter.displayName = "CardFooter"
export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent }
@@ -0,0 +1,30 @@
"use client"
import * as React from "react"
import * as CheckboxPrimitive from "@radix-ui/react-checkbox"
import { Check } from "lucide-react"
import { cn } from "@/lib/utils"
const Checkbox = React.forwardRef<
React.ElementRef<typeof CheckboxPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof CheckboxPrimitive.Root>
>(({ className, ...props }, ref) => (
<CheckboxPrimitive.Root
ref={ref}
className={cn(
"peer h-4 w-4 shrink-0 rounded-sm border border-primary ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground",
className
)}
{...props}
>
<CheckboxPrimitive.Indicator
className={cn("flex items-center justify-center text-current")}
>
<Check className="h-4 w-4" />
</CheckboxPrimitive.Indicator>
</CheckboxPrimitive.Root>
))
Checkbox.displayName = CheckboxPrimitive.Root.displayName
export { Checkbox }
@@ -0,0 +1,25 @@
import * as React from "react"
import { cn } from "@/lib/utils"
export interface InputProps
extends React.InputHTMLAttributes<HTMLInputElement> {}
const Input = React.forwardRef<HTMLInputElement, InputProps>(
({ className, type, ...props }, ref) => {
return (
<input
type={type}
className={cn(
"flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
className
)}
ref={ref}
{...props}
/>
)
}
)
Input.displayName = "Input"
export { Input }
@@ -0,0 +1,26 @@
"use client"
import * as React from "react"
import * as LabelPrimitive from "@radix-ui/react-label"
import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const labelVariants = cva(
"text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
)
const Label = React.forwardRef<
React.ElementRef<typeof LabelPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof LabelPrimitive.Root> &
VariantProps<typeof labelVariants>
>(({ className, ...props }, ref) => (
<LabelPrimitive.Root
ref={ref}
className={cn(labelVariants(), className)}
{...props}
/>
))
Label.displayName = LabelPrimitive.Root.displayName
export { Label }
+6
View File
@@ -0,0 +1,6 @@
import { type ClassValue, clsx } from "clsx"
import { twMerge } from "tailwind-merge"
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs))
}