diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..66b684f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,38 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + - master + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.12.1 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Check formatting + run: pnpm run format:check + + - name: Type check + run: pnpm run type-check + + - name: Build + run: pnpm run build diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 0000000..5ee7abd --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1 @@ +pnpm exec lint-staged diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..7225824 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,8 @@ +{ + "semi": true, + "trailingComma": "es5", + "singleQuote": true, + "printWidth": 80, + "tabWidth": 2, + "useTabs": false +} diff --git a/package.json b/package.json index e1e0d5f..4a64b1b 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,17 @@ "start": "node dist/index.js", "clean": "rm -rf dist", "prepublishOnly": "pnpm run build", + "prepare": "husky", + "format": "prettier --write \"src/**/*.{ts,json}\" \"*.{json,md}\"", + "format:check": "prettier --check \"src/**/*.{ts,json}\" \"*.{json,md}\"", + "type-check": "tsc --noEmit", "test": "echo \"Error: no test specified\" && exit 1" }, + "lint-staged": { + "*.{ts,json,md}": [ + "prettier --write" + ] + }, "keywords": [ "firecrawl", "cli", @@ -44,6 +53,9 @@ "packageManager": "pnpm@10.12.1", "devDependencies": { "@types/node": "^20.0.0", + "husky": "^9.0.0", + "lint-staged": "^15.0.0", + "prettier": "^3.0.0", "typescript": "^5.0.0" }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 511707a..b0be08b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,15 @@ importers: '@types/node': specifier: ^20.0.0 version: 20.19.27 + husky: + specifier: ^9.0.0 + version: 9.1.7 + lint-staged: + specifier: ^15.0.0 + version: 15.5.2 + prettier: + specifier: ^3.0.0 + version: 3.7.4 typescript: specifier: ^5.0.0 version: 5.9.3 @@ -31,24 +40,72 @@ packages: '@types/node@20.19.27': resolution: {integrity: sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug==} + ansi-escapes@7.2.0: + resolution: {integrity: sha512-g6LhBsl+GBPRWGWsBtutpzBYuIIdBkLEvad5C/va/74Db018+5TZiyA26cZJAr3Rft5lprVqOIPxf5Vid6tqAw==} + engines: {node: '>=18'} + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} axios@1.13.2: resolution: {integrity: sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==} + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + cli-cursor@5.0.0: + resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} + engines: {node: '>=18'} + + cli-truncate@4.0.0: + resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==} + engines: {node: '>=18'} + + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} + commander@13.1.0: + resolution: {integrity: sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==} + engines: {node: '>=18'} + commander@14.0.2: resolution: {integrity: sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==} engines: {node: '>=20'} + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -57,6 +114,13 @@ packages: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + + environment@1.1.0: + resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} + engines: {node: '>=18'} + es-define-property@1.0.1: resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} engines: {node: '>= 0.4'} @@ -73,6 +137,17 @@ packages: resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} engines: {node: '>= 0.4'} + eventemitter3@5.0.1: + resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + + execa@8.0.1: + resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==} + engines: {node: '>=16.17'} + + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + follow-redirects@1.15.11: resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} engines: {node: '>=4.0'} @@ -89,6 +164,10 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + get-east-asian-width@1.4.0: + resolution: {integrity: sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==} + engines: {node: '>=18'} + get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -97,6 +176,10 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-stream@8.0.1: + resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} + engines: {node: '>=16'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -113,10 +196,62 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + human-signals@5.0.0: + resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} + engines: {node: '>=16.17.0'} + + husky@9.1.7: + resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} + engines: {node: '>=18'} + hasBin: true + + is-fullwidth-code-point@4.0.0: + resolution: {integrity: sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==} + engines: {node: '>=12'} + + is-fullwidth-code-point@5.1.0: + resolution: {integrity: sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==} + engines: {node: '>=18'} + + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + + is-stream@3.0.0: + resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + lilconfig@3.1.3: + resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} + engines: {node: '>=14'} + + lint-staged@15.5.2: + resolution: {integrity: sha512-YUSOLq9VeRNAo/CTaVmhGDKG+LBtA8KF1X4K5+ykMSwWST1vDxJRB2kv2COgLb1fvpCo+A/y9A0G0znNVmdx4w==} + engines: {node: '>=18.12.0'} + hasBin: true + + listr2@8.3.3: + resolution: {integrity: sha512-LWzX2KsqcB1wqQ4AHgYb4RsDXauQiqhjLk+6hjbaeHG4zpjjVAB6wC/gz6X0l+Du1cN3pUB5ZlrvTbhGSNnUQQ==} + engines: {node: '>=18.0.0'} + + log-update@6.1.0: + resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} + engines: {node: '>=18'} + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} @@ -125,9 +260,101 @@ packages: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mimic-fn@4.0.0: + resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} + engines: {node: '>=12'} + + mimic-function@5.0.1: + resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} + engines: {node: '>=18'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + npm-run-path@5.3.0: + resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + onetime@6.0.0: + resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} + engines: {node: '>=12'} + + onetime@7.0.0: + resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} + engines: {node: '>=18'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-key@4.0.0: + resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==} + engines: {node: '>=12'} + + picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + + pidtree@0.6.0: + resolution: {integrity: sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==} + engines: {node: '>=0.10'} + hasBin: true + + prettier@3.7.4: + resolution: {integrity: sha512-v6UNi1+3hSlVvv8fSaoUbggEM5VErKmmpGA7Pl3HF8V6uKY7rvClBOJlH6yNwQtfTueNkGVpOv/mtWL9L4bgRA==} + engines: {node: '>=14'} + hasBin: true + proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + restore-cursor@5.1.0: + resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} + engines: {node: '>=18'} + + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + slice-ansi@5.0.0: + resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==} + engines: {node: '>=12'} + + slice-ansi@7.1.2: + resolution: {integrity: sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==} + engines: {node: '>=18'} + + string-argv@0.3.2: + resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} + engines: {node: '>=0.6.19'} + + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + + strip-final-newline@3.0.0: + resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} + engines: {node: '>=12'} + + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + typescript-event-target@1.1.2: resolution: {integrity: sha512-TvkrTUpv7gCPlcnSoEwUVUBwsdheKm+HF5u2tPAKubkIGMfovdSizCTaZRY/NhR8+Ijy8iZZUapbVQAsNrkFrw==} @@ -139,6 +366,20 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + + yaml@2.8.2: + resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} + engines: {node: '>= 14.6'} + hasBin: true + zod-to-json-schema@3.25.1: resolution: {integrity: sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==} peerDependencies: @@ -162,6 +403,14 @@ snapshots: dependencies: undici-types: 6.21.0 + ansi-escapes@7.2.0: + dependencies: + environment: 1.1.0 + + ansi-regex@6.2.2: {} + + ansi-styles@6.2.3: {} + asynckit@0.4.0: {} axios@1.13.2: @@ -172,17 +421,46 @@ snapshots: transitivePeerDependencies: - debug + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 function-bind: 1.1.2 + chalk@5.6.2: {} + + cli-cursor@5.0.0: + dependencies: + restore-cursor: 5.1.0 + + cli-truncate@4.0.0: + dependencies: + slice-ansi: 5.0.0 + string-width: 7.2.0 + + colorette@2.0.20: {} + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 + commander@13.1.0: {} + commander@14.0.2: {} + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + delayed-stream@1.0.0: {} dunder-proto@1.0.1: @@ -191,6 +469,10 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + emoji-regex@10.6.0: {} + + environment@1.1.0: {} + es-define-property@1.0.1: {} es-errors@1.3.0: {} @@ -206,6 +488,24 @@ snapshots: has-tostringtag: 1.0.2 hasown: 2.0.2 + eventemitter3@5.0.1: {} + + execa@8.0.1: + dependencies: + cross-spawn: 7.0.6 + get-stream: 8.0.1 + human-signals: 5.0.0 + is-stream: 3.0.0 + merge-stream: 2.0.0 + npm-run-path: 5.3.0 + onetime: 6.0.0 + signal-exit: 4.1.0 + strip-final-newline: 3.0.0 + + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + follow-redirects@1.15.11: {} form-data@4.0.5: @@ -218,6 +518,8 @@ snapshots: function-bind@1.1.2: {} + get-east-asian-width@1.4.0: {} + get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -236,6 +538,8 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + get-stream@8.0.1: {} + gopd@1.2.0: {} has-symbols@1.1.0: {} @@ -248,22 +552,162 @@ snapshots: dependencies: function-bind: 1.1.2 + human-signals@5.0.0: {} + + husky@9.1.7: {} + + is-fullwidth-code-point@4.0.0: {} + + is-fullwidth-code-point@5.1.0: + dependencies: + get-east-asian-width: 1.4.0 + + is-number@7.0.0: {} + + is-stream@3.0.0: {} + + isexe@2.0.0: {} + + lilconfig@3.1.3: {} + + lint-staged@15.5.2: + dependencies: + chalk: 5.6.2 + commander: 13.1.0 + debug: 4.4.3 + execa: 8.0.1 + lilconfig: 3.1.3 + listr2: 8.3.3 + micromatch: 4.0.8 + pidtree: 0.6.0 + string-argv: 0.3.2 + yaml: 2.8.2 + transitivePeerDependencies: + - supports-color + + listr2@8.3.3: + dependencies: + cli-truncate: 4.0.0 + colorette: 2.0.20 + eventemitter3: 5.0.1 + log-update: 6.1.0 + rfdc: 1.4.1 + wrap-ansi: 9.0.2 + + log-update@6.1.0: + dependencies: + ansi-escapes: 7.2.0 + cli-cursor: 5.0.0 + slice-ansi: 7.1.2 + strip-ansi: 7.1.2 + wrap-ansi: 9.0.2 + math-intrinsics@1.1.0: {} + merge-stream@2.0.0: {} + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + mime-db@1.52.0: {} mime-types@2.1.35: dependencies: mime-db: 1.52.0 + mimic-fn@4.0.0: {} + + mimic-function@5.0.1: {} + + ms@2.1.3: {} + + npm-run-path@5.3.0: + dependencies: + path-key: 4.0.0 + + onetime@6.0.0: + dependencies: + mimic-fn: 4.0.0 + + onetime@7.0.0: + dependencies: + mimic-function: 5.0.1 + + path-key@3.1.1: {} + + path-key@4.0.0: {} + + picomatch@2.3.1: {} + + pidtree@0.6.0: {} + + prettier@3.7.4: {} + proxy-from-env@1.1.0: {} + restore-cursor@5.1.0: + dependencies: + onetime: 7.0.0 + signal-exit: 4.1.0 + + rfdc@1.4.1: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + signal-exit@4.1.0: {} + + slice-ansi@5.0.0: + dependencies: + ansi-styles: 6.2.3 + is-fullwidth-code-point: 4.0.0 + + slice-ansi@7.1.2: + dependencies: + ansi-styles: 6.2.3 + is-fullwidth-code-point: 5.1.0 + + string-argv@0.3.2: {} + + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.4.0 + strip-ansi: 7.1.2 + + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + + strip-final-newline@3.0.0: {} + + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + typescript-event-target@1.1.2: {} typescript@5.9.3: {} undici-types@6.21.0: {} + which@2.0.2: + dependencies: + isexe: 2.0.0 + + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.1.2 + + yaml@2.8.2: {} + zod-to-json-schema@3.25.1(zod@3.25.76): dependencies: zod: 3.25.76 diff --git a/src/commands/config.ts b/src/commands/config.ts index 83b4879..5060543 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -18,7 +18,9 @@ function promptInput(question: string, defaultValue?: string): Promise { output: process.stdout, }); - const promptText = defaultValue ? `${question} [${defaultValue}]: ` : `${question} `; + const promptText = defaultValue + ? `${question} [${defaultValue}]: ` + : `${question} `; return new Promise((resolve) => { rl.question(promptText, (answer) => { @@ -37,7 +39,7 @@ export async function configure(): Promise { // Prompt for API URL with default let url = await promptInput('Enter API URL', DEFAULT_API_URL); - + // Ensure URL doesn't end with trailing slash url = url.replace(/\/$/, ''); @@ -58,23 +60,24 @@ export async function configure(): Promise { const normalizedUrl = url.trim().replace(/\/$/, ''); try { - saveCredentials({ + saveCredentials({ apiKey: key.trim(), - apiUrl: normalizedUrl + apiUrl: normalizedUrl, }); console.log('\n✓ Configuration saved successfully'); console.log(` API URL: ${normalizedUrl}`); console.log(` Stored in: ${getConfigDirectoryPath()}`); - + // Update global config - updateConfig({ + updateConfig({ apiKey: key.trim(), - apiUrl: normalizedUrl + apiUrl: normalizedUrl, }); } catch (error) { - console.error('Error saving configuration:', error instanceof Error ? error.message : 'Unknown error'); + console.error( + 'Error saving configuration:', + error instanceof Error ? error.message : 'Unknown error' + ); process.exit(1); } } - - diff --git a/src/commands/credit-usage.ts b/src/commands/credit-usage.ts index 03249a1..b0200a2 100644 --- a/src/commands/credit-usage.ts +++ b/src/commands/credit-usage.ts @@ -31,7 +31,9 @@ export interface CreditUsageOptions { /** * Execute the credit usage command */ -export async function executeCreditUsage(options: CreditUsageOptions = {}): Promise { +export async function executeCreditUsage( + options: CreditUsageOptions = {} +): Promise { try { // Update global config if API key is provided via options if (options.apiKey) { @@ -41,7 +43,7 @@ export async function executeCreditUsage(options: CreditUsageOptions = {}): Prom // Get config and validate API key const config = getConfig(); validateConfig(config.apiKey); - + const apiUrl = config.apiUrl || 'https://api.firecrawl.dev'; const apiKey = config.apiKey!; @@ -50,18 +52,20 @@ export async function executeCreditUsage(options: CreditUsageOptions = {}): Prom const response = await fetch(url, { method: 'GET', headers: { - 'Authorization': `Bearer ${apiKey}`, + Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, }); if (!response.ok) { const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.error || `HTTP ${response.status}: ${response.statusText}`); + throw new Error( + errorData.error || `HTTP ${response.status}: ${response.statusText}` + ); } const result = await response.json(); - + // Extract data from response (handle both { data: {...} } and direct data formats) const data = result.data || result; @@ -76,10 +80,9 @@ export async function executeCreditUsage(options: CreditUsageOptions = {}): Prom }; } catch (error: any) { // Handle different error formats - const errorMessage = error?.message - || error?.toString() - || 'Unknown error occurred'; - + const errorMessage = + error?.message || error?.toString() || 'Unknown error occurred'; + return { success: false, error: errorMessage, @@ -99,10 +102,10 @@ function formatReadable(data: CreditUsageResult['data']): string { }; const lines: string[] = []; - + // Main credit info lines.push(`Remaining Credits: ${formatNumber(data.remainingCredits)}`); - + if (data.planCredits > 0) { const usedCredits = data.planCredits - data.remainingCredits; const usagePercent = ((usedCredits / data.planCredits) * 100).toFixed(1); @@ -115,14 +118,16 @@ function formatReadable(data: CreditUsageResult['data']): string { const startDate = new Date(data.billingPeriodStart); const endDate = new Date(data.billingPeriodEnd); const formatDate = (date: Date): string => { - return date.toLocaleDateString('en-US', { - year: 'numeric', - month: 'short', - day: 'numeric' + return date.toLocaleDateString('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric', }); }; lines.push(''); - lines.push(`Billing Period: ${formatDate(startDate)} - ${formatDate(endDate)}`); + lines.push( + `Billing Period: ${formatDate(startDate)} - ${formatDate(endDate)}` + ); } return lines.join('\n') + '\n'; @@ -131,9 +136,11 @@ function formatReadable(data: CreditUsageResult['data']): string { /** * Handle credit usage command output */ -export async function handleCreditUsageCommand(options: CreditUsageOptions = {}): Promise { +export async function handleCreditUsageCommand( + options: CreditUsageOptions = {} +): Promise { const result = await executeCreditUsage(options); - + if (!result.success) { console.error('Error:', result.error); process.exit(1); @@ -148,20 +155,20 @@ export async function handleCreditUsageCommand(options: CreditUsageOptions = {}) // Use JSON format if --json flag is set if (options.json) { try { - outputContent = options.pretty + outputContent = options.pretty ? JSON.stringify({ success: true, data: result.data }, null, 2) : JSON.stringify({ success: true, data: result.data }); } catch (error) { - outputContent = JSON.stringify({ + outputContent = JSON.stringify({ error: 'Failed to serialize response', - message: error instanceof Error ? error.message : 'Unknown error' + message: error instanceof Error ? error.message : 'Unknown error', }); } } else { // Default to human-readable format outputContent = formatReadable(result.data); } - + // Write output if (options.output) { const dir = path.dirname(options.output); diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index fac3fc4..020faad 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -11,7 +11,9 @@ import { handleScrapeOutput } from '../utils/output'; /** * Execute the scrape command */ -export async function executeScrape(options: ScrapeOptions): Promise { +export async function executeScrape( + options: ScrapeOptions +): Promise { try { // Update global config if API key is provided via options if (options.apiKey) { @@ -23,7 +25,7 @@ export async function executeScrape(options: ScrapeOptions): Promise { +export async function handleScrapeCommand( + options: ScrapeOptions +): Promise { const result = await executeScrape(options); handleScrapeOutput(result, options.format, options.output, options.pretty); } - diff --git a/src/index.ts b/src/index.ts index 42345ce..6c84075 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,7 +22,10 @@ program .name('firecrawl') .description('CLI tool for Firecrawl web scraping') .version('1.0.0') - .option('-k, --api-key ', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var, or use "firecrawl config")') + .option( + '-k, --api-key ', + 'Firecrawl API key (or set FIRECRAWL_API_KEY env var, or use "firecrawl config")' + ) .allowUnknownOption() // Allow unknown options when URL is passed directly .hook('preAction', (thisCommand, actionCommand) => { // Update global config if API key is provided via global option @@ -39,28 +42,44 @@ function createScrapeCommand(): Command { const scrapeCmd = new Command('scrape') .description('Scrape a URL using Firecrawl') .argument('[url]', 'URL to scrape') - .option('-u, --url ', 'URL to scrape (alternative to positional argument)') + .option( + '-u, --url ', + 'URL to scrape (alternative to positional argument)' + ) .option('-H, --html', 'Output raw HTML (shortcut for --format html)') - .option('-f, --format ', 'Output format: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding', 'markdown') + .option( + '-f, --format ', + 'Output format: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding', + 'markdown' + ) .option('--only-main-content', 'Include only main content', false) - .option('--wait-for ', 'Wait time before scraping in milliseconds', parseInt) + .option( + '--wait-for ', + 'Wait time before scraping in milliseconds', + parseInt + ) .option('--screenshot', 'Take a screenshot', false) .option('--include-tags ', 'Comma-separated list of tags to include') .option('--exclude-tags ', 'Comma-separated list of tags to exclude') - .option('-k, --api-key ', 'Firecrawl API key (overrides global --api-key)') + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) .option('-o, --output ', 'Output file path (default: stdout)') .option('--pretty', 'Pretty print JSON output', false) .action(async (positionalUrl, options) => { // Use positional URL if provided, otherwise use --url option const url = positionalUrl || options.url; if (!url) { - console.error('Error: URL is required. Provide it as argument or use --url option.'); + console.error( + 'Error: URL is required. Provide it as argument or use --url option.' + ); process.exit(1); } - + // Handle --html shortcut flag const format = options.html ? 'html' : options.format; - + const scrapeOptions = parseScrapeOptions({ ...options, url, format }); await handleScrapeCommand(scrapeOptions); }); @@ -81,10 +100,17 @@ program program .command('credit-usage') .description('Get team credit usage information') - .option('-k, --api-key ', 'Firecrawl API key (overrides global --api-key)') + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) .option('-o, --output ', 'Output file path (default: stdout)') .option('--json', 'Output as JSON format', false) - .option('--pretty', 'Pretty print JSON output (only applies with --json)', false) + .option( + '--pretty', + 'Pretty print JSON output (only applies with --json)', + false + ) .action(async (options) => { await handleCreditUsageCommand(options); }); @@ -96,14 +122,23 @@ const args = process.argv.slice(2); if (args.length > 0 && !args[0].startsWith('-') && isUrl(args[0])) { // Treat as scrape command with URL - reuse commander's parsing const url = normalizeUrl(args[0]); - + // Modify argv to include scrape command with URL as positional argument // This allows commander to parse it normally with all hooks and options - const modifiedArgv = [process.argv[0], process.argv[1], 'scrape', url, ...args.slice(1)]; - + const modifiedArgv = [ + process.argv[0], + process.argv[1], + 'scrape', + url, + ...args.slice(1), + ]; + // Parse using the main program (which includes hooks and global options) program.parseAsync(modifiedArgv).catch((error) => { - console.error('Error:', error instanceof Error ? error.message : 'Unknown error'); + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error' + ); process.exit(1); }); } else { diff --git a/src/types/scrape.ts b/src/types/scrape.ts index 9a1a754..508a25e 100644 --- a/src/types/scrape.ts +++ b/src/types/scrape.ts @@ -43,4 +43,3 @@ export interface ScrapeResult { data?: any; error?: string; } - diff --git a/src/utils/client.ts b/src/utils/client.ts index 6bf2075..51da231 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -13,9 +13,13 @@ let clientInstance: Firecrawl | null = null; * Get or create the Firecrawl client instance * Uses global configuration if available, otherwise creates with provided options */ -export function getClient(options?: Partial): Firecrawl { +export function getClient( + options?: Partial +): Firecrawl { // Helper to convert null to undefined and ensure we have a string or undefined - const normalizeApiKey = (value: string | null | undefined): string | undefined => + const normalizeApiKey = ( + value: string | null | undefined + ): string | undefined => value === null || value === undefined ? undefined : value; // If options provided, create a new instance (useful for command-specific overrides) @@ -23,11 +27,11 @@ export function getClient(options?: Partial): Firecrawl const config = getConfig(); const apiKey = normalizeApiKey(options.apiKey) ?? config.apiKey; const apiUrl = normalizeApiKey(options.apiUrl) ?? config.apiUrl; - + // Normalize apiKey for validation (convert null to undefined) const normalizedApiKey = apiKey === null ? undefined : apiKey; validateConfig(normalizedApiKey); - + const clientOptions: FirecrawlClientOptions = { apiKey: normalizedApiKey || undefined, apiUrl: apiUrl === null ? undefined : apiUrl, @@ -35,7 +39,7 @@ export function getClient(options?: Partial): Firecrawl maxRetries: options.maxRetries ?? config.maxRetries, backoffFactor: options.backoffFactor ?? config.backoffFactor, }; - + return new Firecrawl(clientOptions); } @@ -43,7 +47,7 @@ export function getClient(options?: Partial): Firecrawl if (!clientInstance) { const config = getConfig(); validateConfig(config.apiKey); - + const clientOptions: FirecrawlClientOptions = { apiKey: config.apiKey || undefined, apiUrl: config.apiUrl || undefined, @@ -51,7 +55,7 @@ export function getClient(options?: Partial): Firecrawl maxRetries: config.maxRetries, backoffFactor: config.backoffFactor, }; - + clientInstance = new Firecrawl(clientOptions); } @@ -67,7 +71,7 @@ export function initializeClient(config?: Partial): Firecrawl { const { initializeConfig } = require('./config'); initializeConfig(config); } - + // Reset instance to force recreation with new config clientInstance = null; return getClient(); @@ -79,4 +83,3 @@ export function initializeClient(config?: Partial): Firecrawl { export function resetClient(): void { clientInstance = null; } - diff --git a/src/utils/config.ts b/src/utils/config.ts index 410220b..f75e72d 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -25,10 +25,16 @@ let globalConfig: GlobalConfig = {}; export function initializeConfig(config: Partial = {}): void { // Priority: provided config > env vars > stored credentials const storedCredentials = loadCredentials(); - + globalConfig = { - apiKey: config.apiKey || process.env.FIRECRAWL_API_KEY || storedCredentials?.apiKey, - apiUrl: config.apiUrl || process.env.FIRECRAWL_API_URL || storedCredentials?.apiUrl, + apiKey: + config.apiKey || + process.env.FIRECRAWL_API_KEY || + storedCredentials?.apiKey, + apiUrl: + config.apiUrl || + process.env.FIRECRAWL_API_URL || + storedCredentials?.apiUrl, timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, backoffFactor: config.backoffFactor, @@ -60,7 +66,7 @@ export function getApiKey(providedKey?: string): string | undefined { if (providedKey) return providedKey; if (globalConfig.apiKey) return globalConfig.apiKey; if (process.env.FIRECRAWL_API_KEY) return process.env.FIRECRAWL_API_KEY; - + // Fallback to stored credentials if not already loaded const storedCredentials = loadCredentials(); return storedCredentials?.apiKey; @@ -84,4 +90,3 @@ export function validateConfig(apiKey?: string): void { export function resetConfig(): void { globalConfig = {}; } - diff --git a/src/utils/credentials.ts b/src/utils/credentials.ts index 9d3e788..fdb7d79 100644 --- a/src/utils/credentials.ts +++ b/src/utils/credentials.ts @@ -21,7 +21,12 @@ function getConfigDir(): string { switch (platform) { case 'darwin': // macOS - return path.join(homeDir, 'Library', 'Application Support', 'firecrawl-cli'); + return path.join( + homeDir, + 'Library', + 'Application Support', + 'firecrawl-cli' + ); case 'win32': // Windows return path.join(homeDir, 'AppData', 'Roaming', 'firecrawl-cli'); default: // Linux and others @@ -83,7 +88,7 @@ export function saveCredentials(credentials: StoredCredentials): void { try { ensureConfigDir(); const credentialsPath = getCredentialsPath(); - + // Read existing credentials and merge const existing = loadCredentials(); const merged: StoredCredentials = { @@ -92,16 +97,14 @@ export function saveCredentials(credentials: StoredCredentials): void { }; // Write to file - fs.writeFileSync( - credentialsPath, - JSON.stringify(merged, null, 2), - 'utf-8' - ); + fs.writeFileSync(credentialsPath, JSON.stringify(merged, null, 2), 'utf-8'); // Set secure permissions setSecurePermissions(credentialsPath); } catch (error) { - throw new Error(`Failed to save credentials: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error( + `Failed to save credentials: ${error instanceof Error ? error.message : 'Unknown error'}` + ); } } @@ -115,7 +118,9 @@ export function deleteCredentials(): void { fs.unlinkSync(credentialsPath); } } catch (error) { - throw new Error(`Failed to delete credentials: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error( + `Failed to delete credentials: ${error instanceof Error ? error.message : 'Unknown error'}` + ); } } @@ -125,4 +130,3 @@ export function deleteCredentials(): void { export function getConfigDirectoryPath(): string { return getConfigDir(); } - diff --git a/src/utils/options.ts b/src/utils/options.ts index 87bb627..2c32070 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -14,11 +14,14 @@ export function parseScrapeOptions(options: any): ScrapeOptions { onlyMainContent: options.onlyMainContent, waitFor: options.waitFor, screenshot: options.screenshot, - includeTags: options.includeTags ? options.includeTags.split(',').map((t: string) => t.trim()) : undefined, - excludeTags: options.excludeTags ? options.excludeTags.split(',').map((t: string) => t.trim()) : undefined, + includeTags: options.includeTags + ? options.includeTags.split(',').map((t: string) => t.trim()) + : undefined, + excludeTags: options.excludeTags + ? options.excludeTags.split(',').map((t: string) => t.trim()) + : undefined, apiKey: options.apiKey, output: options.output, pretty: options.pretty, }; } - diff --git a/src/utils/output.ts b/src/utils/output.ts index 65f6792..d913f8d 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -19,22 +19,22 @@ function extractContent(data: any, format?: ScrapeFormat): string | null { if (format === 'html' || format === 'rawHtml') { return data.html || data.rawHtml || data[format] || null; } - + // Handle markdown format if (format === 'markdown') { return data.markdown || data[format] || null; } - + // Handle links format if (format === 'links') { return data.links || data[format] || null; } - + // Handle images format if (format === 'images') { return data.images || data[format] || null; } - + // Handle summary format if (format === 'summary') { return data.summary || data[format] || null; @@ -104,7 +104,14 @@ export function handleScrapeOutput( } // Text formats that should output raw content (curl-like) - const rawTextFormats: ScrapeFormat[] = ['html', 'rawHtml', 'markdown', 'links', 'images', 'summary']; + const rawTextFormats: ScrapeFormat[] = [ + 'html', + 'rawHtml', + 'markdown', + 'links', + 'images', + 'summary', + ]; const shouldOutputRaw = format && rawTextFormats.includes(format); if (shouldOutputRaw) { @@ -120,19 +127,18 @@ export function handleScrapeOutput( // Always stringify the entire data object to ensure valid JSON let jsonContent: string; try { - jsonContent = pretty + jsonContent = pretty ? JSON.stringify(result.data, null, 2) : JSON.stringify(result.data); } catch (error) { // If stringification fails, try to create a minimal error response - jsonContent = JSON.stringify({ + jsonContent = JSON.stringify({ error: 'Failed to serialize response', - message: error instanceof Error ? error.message : 'Unknown error' + message: error instanceof Error ? error.message : 'Unknown error', }); } - + // Ensure clean JSON output (no extra newlines or text before JSON) // Output directly to stdout without any prefix writeOutput(jsonContent, outputPath, !!outputPath); } - diff --git a/src/utils/url.ts b/src/utils/url.ts index 6ad0de7..4623c36 100644 --- a/src/utils/url.ts +++ b/src/utils/url.ts @@ -15,15 +15,16 @@ export function isUrl(str: string): boolean { return true; // Assume valid if it starts with http:// or https:// } } - + // Check if it looks like a domain (has dots and valid characters) // Exclude common commands and flags if (str.includes('.') && !str.startsWith('-') && !str.includes(' ')) { // Basic domain validation: contains at least one dot and valid characters - const domainPattern = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}(\/.*)?$/; + const domainPattern = + /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}(\/.*)?$/; return domainPattern.test(str); } - + return false; } @@ -36,4 +37,3 @@ export function normalizeUrl(url: string): string { } return `https://${url}`; } - diff --git a/tsconfig.json b/tsconfig.json index 147aded..d6d9394 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,4 +18,3 @@ "include": ["src/**/*"], "exclude": ["node_modules", "dist", "**/*.test.ts"] } -