diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index 593a039e..a96dc2f3 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,15 +1,14 @@ { "name": "@mendable/firecrawl-js", - "version": "1.22.0", + "version": "1.22.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "1.22.0", + "version": "1.22.1", "license": "MIT", "dependencies": { - "axios": "^1.6.8", "typescript-event-target": "^1.1.1", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.0" @@ -18,10 +17,11 @@ "@jest/globals": "^29.7.0", "@types/axios": "^0.14.0", "@types/dotenv": "^8.2.0", - "@types/jest": "^29.5.12", + "@types/jest": "^29.5.14", "@types/mocha": "^10.0.6", - "@types/node": "^20.12.12", + "@types/node": "^20.17.30", "@types/uuid": "^9.0.8", + "axios": "^1.8.4", "dotenv": "^16.4.5", "jest": "^29.7.0", "ts-jest": "^29.2.2", @@ -1812,10 +1812,11 @@ } }, "node_modules/@types/jest": { - "version": "29.5.12", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.12.tgz", - "integrity": "sha512-eDC8bTvT/QhYdxJAulQikueigY5AsdBRH2yDKW3yveW7svY3+DzN84/2NUgkw10RTiJbWqZrTtoGVdYlvFJdLw==", + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", "dev": true, + "license": "MIT", "dependencies": { "expect": "^29.0.0", "pretty-format": "^29.0.0" @@ -1949,12 +1950,15 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true }, "node_modules/axios": { - "version": "1.6.8", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", - "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz", + "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==", + "dev": true, + "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -2351,6 +2355,7 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -2467,6 +2472,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, "engines": { "node": ">=0.4.0" } @@ -2784,6 +2790,7 @@ "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "dev": true, "funding": [ { "type": "individual", @@ -2831,6 +2838,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -4111,6 +4119,7 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, "engines": { "node": ">= 0.6" } @@ -4119,6 +4128,7 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, "dependencies": { "mime-db": "1.52.0" }, @@ -4507,7 +4517,8 @@ "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true }, "node_modules/punycode": { "version": "2.3.1", diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 7c64b33e..bf7ed81d 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.22.1", + "version": "1.23.0", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -25,7 +25,6 @@ "author": "Mendable.ai", "license": "MIT", "dependencies": { - "axios": "^1.6.8", "typescript-event-target": "^1.1.1", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.0" @@ -38,10 +37,11 @@ "@jest/globals": "^29.7.0", "@types/axios": "^0.14.0", "@types/dotenv": "^8.2.0", - "@types/jest": "^29.5.12", + "@types/jest": "^29.5.14", "@types/mocha": "^10.0.6", - "@types/node": "^20.12.12", + "@types/node": "^20.17.30", "@types/uuid": "^9.0.8", + "axios": "^1.8.4", "dotenv": "^16.4.5", "jest": "^29.7.0", "ts-jest": "^29.2.2", diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/snips/change-tracking.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/snips/change-tracking.test.ts new file mode 100644 index 00000000..835582ce --- /dev/null +++ b/apps/js-sdk/firecrawl/src/__tests__/v1/snips/change-tracking.test.ts @@ -0,0 +1,105 @@ +import axios from 'axios'; +import FirecrawlApp from '../../../../src/index'; + +jest.mock('axios'); +const mockedAxios = axios as jest.Mocked; + +describe('Change Tracking Tests', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should support basic change tracking format', async () => { + mockedAxios.post.mockResolvedValueOnce({ + status: 200, + data: { + success: true, + data: { + markdown: 'Test markdown content', + changeTracking: { + previousScrapeAt: '2023-01-01T00:00:00Z', + changeStatus: 'changed', + visibility: 'visible' + } + } + } + }); + + const app = new FirecrawlApp({ apiKey: process.env.TEST_API_KEY || 'dummy-api-key-for-testing' }); + const result = await app.scrapeUrl('https://example.com', { + formats: ['markdown', 'changeTracking'] + }); + + expect(mockedAxios.post).toHaveBeenCalledTimes(1); + expect(mockedAxios.post.mock.calls[0][1].formats).toContain('changeTracking'); + + expect(result).toHaveProperty('changeTracking'); + expect(result.changeTracking?.previousScrapeAt).toBe('2023-01-01T00:00:00Z'); + expect(result.changeTracking?.changeStatus).toBe('changed'); + expect(result.changeTracking?.visibility).toBe('visible'); + }); + + it('should support change tracking options with git-diff and json modes', async () => { + mockedAxios.post.mockResolvedValueOnce({ + status: 200, + data: { + success: true, + data: { + markdown: 'Test markdown content', + changeTracking: { + previousScrapeAt: '2023-01-01T00:00:00Z', + changeStatus: 'changed', + visibility: 'visible', + diff: { + text: '@@ -1,1 +1,1 @@\n-old content\n+new content', + json: { + files: [{ + from: null, + to: null, + chunks: [{ + content: '@@ -1,1 +1,1 @@', + changes: [{ + type: 'del', + content: '-old content', + del: true, + ln: 1 + }, { + type: 'add', + content: '+new content', + add: true, + ln: 1 + }] + }] + }] + } + }, + json: { + title: { + previous: 'Old Title', + current: 'New Title' + } + } + } + } + } + }); + + const app = new FirecrawlApp({ apiKey: process.env.TEST_API_KEY || 'dummy-api-key-for-testing' }); + const result = await app.scrapeUrl('https://example.com', { + formats: ['markdown', 'changeTracking'], + changeTrackingOptions: { + modes: ['git-diff', 'json'], + schema: { type: 'object', properties: { title: { type: 'string' } } } + } + }); + + expect(mockedAxios.post).toHaveBeenCalledTimes(1); + expect(mockedAxios.post.mock.calls[0][1].formats).toContain('changeTracking'); + expect(mockedAxios.post.mock.calls[0][1].changeTrackingOptions.modes).toEqual(['git-diff', 'json']); + + expect(result).toHaveProperty('changeTracking'); + expect(result.changeTracking?.diff?.text).toBe('@@ -1,1 +1,1 @@\n-old content\n+new content'); + expect(result.changeTracking?.json?.title.previous).toBe('Old Title'); + expect(result.changeTracking?.json?.title.current).toBe('New Title'); + }); +}); diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 7ba44b70..28fbe075 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -74,7 +74,7 @@ export interface FirecrawlDocument; }; }; + json?: any; }; // v1 search only title?: string; @@ -160,6 +161,11 @@ export interface ScrapeParams