diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 44726f4eb418..685ec644b598 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -100,6 +100,69 @@ jobs: env: YARN_IGNORE_NODE: 1 + bun_test: + name: Bun Compatibility + if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:')) + runs-on: ubuntu-22.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Use Node.js 24 + uses: actions/setup-node@v6 + with: + node-version: 24 + package-manager-cache: false + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Activate cache for Node.js 24 + uses: actions/setup-node@v6 + with: + cache: 'yarn' + + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v5 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- + + - name: Install Dependencies + run: yarn + env: + YARN_IGNORE_NODE: 1 + + - name: Build + run: yarn ci:build + env: + YARN_IGNORE_NODE: 1 + + - name: Bun compatibility tests + run: | + bun run vitest run \ + test/core/runtime.test.ts \ + test/core/proxy_configuration.test.ts \ + test/core/base_http_client.test.ts \ + test/core/crawlers/http_crawler.test.ts \ + test/core/storages/ \ + test/core/session_pool/ \ + test/core/router.test.ts \ + test/core/error_tracker.test.ts + env: + YARN_IGNORE_NODE: 1 + docs: name: Docs build if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master') diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.ts index be75c6dafb08..44e330dbc18d 100644 --- a/packages/core/src/http_clients/got-scraping-http-client.ts +++ b/packages/core/src/http_clients/got-scraping-http-client.ts @@ -1,4 +1,4 @@ -import { gotScraping } from '@crawlee/utils'; +import { gotScraping, warnIfBunRuntime } from '@crawlee/utils'; // @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood import type { Options, PlainResponse } from 'got-scraping'; @@ -15,6 +15,10 @@ import type { * A HTTP client implementation based on the `got-scraping` library. */ export class GotScrapingHttpClient implements BaseHttpClient { + constructor() { + warnIfBunRuntime(); + } + /** * @inheritDoc */ diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 77ff08d8832e..9dcece24b3b5 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -13,6 +13,7 @@ export * from './internals/iterables'; export * from './internals/robots'; export * from './internals/sitemap'; export * from './internals/url'; +export * from './internals/runtime'; export { getCurrentCpuTicksV2 } from './internals/systemInfoV2/cpu-info'; export { getMemoryInfoV2 } from './internals/systemInfoV2/memory-info'; diff --git a/packages/utils/src/internals/runtime.ts b/packages/utils/src/internals/runtime.ts new file mode 100644 index 000000000000..a0db4db60eff --- /dev/null +++ b/packages/utils/src/internals/runtime.ts @@ -0,0 +1,35 @@ +let warnedAboutBun = false; + +/** + * Detects whether the current process is running under the Bun runtime. + */ +export function isBunRuntime(): boolean { + return typeof globalThis !== 'undefined' && 'Bun' in globalThis; +} + +/** + * Logs a one-time warning when `got-scraping` based HTTP client is used under Bun. + * Bun does not fully support `got-scraping`'s tunnel mechanism; users should + * switch to `ImpitHttpClient` from `@crawlee/impit-client` instead. + */ +export function warnIfBunRuntime(logger?: { warning(msg: string, data?: Record): void }): void { + if (!isBunRuntime() || warnedAboutBun) return; + warnedAboutBun = true; + + const message = + 'Detected Bun runtime. GotScrapingHttpClient is not fully compatible with Bun — ' + + 'proxy tunneling and some Node.js stream APIs may not work. ' + + 'Use ImpitHttpClient from @crawlee/impit-client instead: ' + + 'new CheerioCrawler({ httpClient: new ImpitHttpClient() })'; + + if (logger) { + logger.warning(message); + } else { + console.warn(`[crawlee] ${message}`); + } +} + +/** @internal — reset for testing */ +export function _resetBunWarning(): void { + warnedAboutBun = false; +} diff --git a/test/core/runtime.test.ts b/test/core/runtime.test.ts new file mode 100644 index 000000000000..75cdbb5fd038 --- /dev/null +++ b/test/core/runtime.test.ts @@ -0,0 +1,52 @@ +import { _resetBunWarning, isBunRuntime, warnIfBunRuntime } from '@crawlee/utils'; + +afterEach(() => { + _resetBunWarning(); + delete (globalThis as any).Bun; +}); + +describe('isBunRuntime', () => { + test('returns false when Bun global is not present', () => { + delete (globalThis as any).Bun; + expect(isBunRuntime()).toBe(false); + }); + + test('returns true when Bun global is present', () => { + (globalThis as any).Bun = { version: '1.0.0' }; + expect(isBunRuntime()).toBe(true); + }); +}); + +describe('warnIfBunRuntime', () => { + test('does nothing when not running under Bun', () => { + delete (globalThis as any).Bun; + const logger = { warning: vitest.fn() }; + warnIfBunRuntime(logger); + expect(logger.warning).not.toHaveBeenCalled(); + }); + + test('calls logger.warning when running under Bun', () => { + (globalThis as any).Bun = { version: '1.0.0' }; + const logger = { warning: vitest.fn() }; + warnIfBunRuntime(logger); + expect(logger.warning).toHaveBeenCalledTimes(1); + expect(logger.warning).toHaveBeenCalledWith(expect.stringContaining('ImpitHttpClient')); + }); + + test('only warns once even if called multiple times', () => { + (globalThis as any).Bun = { version: '1.0.0' }; + const logger = { warning: vitest.fn() }; + warnIfBunRuntime(logger); + warnIfBunRuntime(logger); + warnIfBunRuntime(logger); + expect(logger.warning).toHaveBeenCalledTimes(1); + }); + + test('falls back to console.warn when no logger is provided', () => { + (globalThis as any).Bun = { version: '1.0.0' }; + const spy = vitest.spyOn(console, 'warn').mockImplementation(() => {}); + warnIfBunRuntime(); + expect(spy).toHaveBeenCalledTimes(1); + expect(spy).toHaveBeenCalledWith(expect.stringContaining('ImpitHttpClient')); + }); +});