feat: add go-sdk

2024-08-04 17:14:55 +08:00
parent 789c6cf5d7
commit 1378ffc138
10 changed files with 1218 additions and 0 deletions
@@ -0,0 +1,83 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/google/uuid"
+	"github.com/mendableai/firecrawl/go-sdk/firecrawl"
+)
+
+func main() {
+	app, err := firecrawl.NewFirecrawlApp("fc-YOUR-API-KEY", "http://localhost:3002")
+	if err != nil {
+		log.Fatalf("Failed to create FirecrawlApp: %v", err)
+	}
+
+	// Scrape a website
+	scrapeResult, err := app.ScrapeURL("firecrawl.dev", nil)
+	if err != nil {
+		log.Fatalf("Failed to scrape URL: %v", err)
+	}
+	fmt.Println(scrapeResult.Markdown)
+
+	// Crawl a website
+	idempotencyKey := uuid.New().String() // optional idempotency key
+	crawlParams := map[string]any{
+		"crawlerOptions": map[string]any{
+			"excludes": []string{"blog/*"},
+		},
+	}
+	crawlResult, err := app.CrawlURL("mendable.ai", crawlParams, true, 2, idempotencyKey)
+	if err != nil {
+		log.Fatalf("Failed to crawl URL: %v", err)
+	}
+	fmt.Println(crawlResult)
+
+	// LLM Extraction using JSON schema
+	jsonSchema := map[string]any{
+		"type": "object",
+		"properties": map[string]any{
+			"top": map[string]any{
+				"type": "array",
+				"items": map[string]any{
+					"type": "object",
+					"properties": map[string]any{
+						"title":       map[string]string{"type": "string"},
+						"points":      map[string]string{"type": "number"},
+						"by":          map[string]string{"type": "string"},
+						"commentsURL": map[string]string{"type": "string"},
+					},
+					"required": []string{"title", "points", "by", "commentsURL"},
+				},
+				"minItems":    5,
+				"maxItems":    5,
+				"description": "Top 5 stories on Hacker News",
+			},
+		},
+		"required": []string{"top"},
+	}
+
+	llmExtractionParams := map[string]any{
+		"extractorOptions": firecrawl.ExtractorOptions{
+			ExtractionSchema: jsonSchema,
+			Mode:             "llm-extraction",
+		},
+		"pageOptions": map[string]any{
+			"onlyMainContent": true,
+		},
+	}
+
+	llmExtractionResult, err := app.ScrapeURL("https://news.ycombinator.com", llmExtractionParams)
+	if err != nil {
+		log.Fatalf("Failed to perform LLM extraction: %v", err)
+	}
+
+	// Pretty print the LLM extraction result
+	jsonResult, err := json.MarshalIndent(llmExtractionResult.LLMExtraction, "", "  ")
+	if err != nil {
+		log.Fatalf("Failed to marshal LLM extraction result: %v", err)
+	}
+	fmt.Println(string(jsonResult))
+}
@@ -0,0 +1,10 @@
+module github.com/mendableai/firecrawl/go-sdk/examples
+
+go 1.22.5
+
+replace github.com/mendableai/firecrawl/go-sdk => ../
+
+require (
+	github.com/google/uuid v1.6.0
+	github.com/mendableai/firecrawl/go-sdk v0.0.0-00010101000000-000000000000
+)
@@ -0,0 +1,12 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=