From 6d75720a89a371fc227fa79cb0bbc47d6e4a4390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=88=90?= Date: Wed, 18 Mar 2026 16:28:26 +0800 Subject: [PATCH] 1 --- server/config/app_config.js | 3 +- server/config/database.js | 1 - server/package-lock.json | 460 ++++++++++++++++++ server/package.json | 2 + server/services/puppeteer/puppeteer_runner.js | 201 ++++++++ .../services/puppeteer/puppeteer_stealth.js | 27 + server/services/puppeteer_runner.js | 148 ------ server/services/task_executor.js | 13 +- 8 files changed, 697 insertions(+), 158 deletions(-) create mode 100644 server/services/puppeteer/puppeteer_runner.js create mode 100644 server/services/puppeteer/puppeteer_stealth.js delete mode 100644 server/services/puppeteer_runner.js diff --git a/server/config/app_config.js b/server/config/app_config.js index 0847018..4ae23b7 100644 --- a/server/config/app_config.js +++ b/server/config/app_config.js @@ -57,7 +57,8 @@ export function get_app_config() { puppeteer_headless: get_bool('PUPPETEER_HEADLESS', false), chrome_executable_path: (get_env('CHROME_EXECUTABLE_PATH') || '').trim() || path.resolve(__dirname, '../../chrome-win/chrome.exe'), log_invoke_action: get_bool('LOG_INVOKE_ACTION', true), - auto_close_browser: get_bool('AUTO_CLOSE_BROWSER', true) + auto_close_browser: get_bool('AUTO_CLOSE_BROWSER', true), + enable_stealth: get_bool('ENABLE_STEALTH', true) } }; diff --git a/server/config/database.js b/server/config/database.js index 921f58d..2edf05f 100644 --- a/server/config/database.js +++ b/server/config/database.js @@ -2,7 +2,6 @@ import { get_app_config } from './app_config.js'; export function get_sequelize_options() { const cfg = get_app_config(); - console.log( 'get_sequelize_options', cfg.mysql ); return { host: cfg.mysql.host, diff --git a/server/package-lock.json b/server/package-lock.json index fd8d24d..6bad497 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -14,6 +14,8 @@ "mysql2": "^3.11.0", "node-cron": "^3.0.3", "puppeteer": "^23.4.1", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2", "sequelize": "^6.37.3" } }, @@ -182,6 +184,15 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "license": "Python-2.0" }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ast-types": { "version": "0.13.4", "resolved": "https://registry.npmmirror.com/ast-types/-/ast-types-0.13.4.tgz", @@ -217,6 +228,12 @@ } } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, "node_modules/bare-events": { "version": "2.8.2", "resolved": "https://registry.npmmirror.com/bare-events/-/bare-events-2.8.2.tgz", @@ -333,6 +350,16 @@ "node": ">=10.0.0" } }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz", @@ -453,6 +480,22 @@ "node": ">=12" } }, + "node_modules/clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmmirror.com/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==", + "license": "MIT", + "dependencies": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmmirror.com/co/-/co-4.6.0.tgz", @@ -497,6 +540,12 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz", @@ -595,6 +644,15 @@ "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==", "license": "MIT" }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/degenerator": { "version": "5.0.1", "resolved": "https://registry.npmmirror.com/degenerator/-/degenerator-5.0.1.tgz", @@ -858,6 +916,27 @@ "pend": "~1.2.0" } }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmmirror.com/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", + "license": "MIT", + "dependencies": { + "for-in": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz", @@ -867,6 +946,26 @@ "node": ">= 0.6" } }, + "node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz", @@ -969,6 +1068,27 @@ "node": ">= 14" } }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmmirror.com/gopd/-/gopd-1.2.0.tgz", @@ -981,6 +1101,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.1.0.tgz", @@ -1183,6 +1309,17 @@ ], "license": "MIT" }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz", @@ -1204,6 +1341,21 @@ "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "license": "MIT" }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmmirror.com/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT" + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmmirror.com/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -1232,6 +1384,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-property": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/is-property/-/is-property-1.0.2.tgz", @@ -1256,6 +1420,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmmirror.com/js-tokens/-/js-tokens-4.0.0.tgz", @@ -1280,6 +1453,18 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "license": "MIT" }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/keygrip": { "version": "1.1.0", "resolved": "https://registry.npmmirror.com/keygrip/-/keygrip-1.1.0.tgz", @@ -1293,6 +1478,18 @@ "node": ">= 0.6" } }, + "node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "license": "MIT", + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/koa": { "version": "2.16.4", "resolved": "https://registry.npmmirror.com/koa/-/koa-2.16.4.tgz", @@ -1394,6 +1591,15 @@ "node": ">= 0.6" } }, + "node_modules/lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmmirror.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -1454,6 +1660,20 @@ "node": ">= 0.6" } }, + "node_modules/merge-deep": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/merge-deep/-/merge-deep-3.0.3.tgz", + "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==", + "license": "MIT", + "dependencies": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/methods": { "version": "1.1.2", "resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz", @@ -1484,12 +1704,46 @@ "node": ">= 0.6" } }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/mitt": { "version": "3.0.1", "resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz", "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", "license": "MIT" }, + "node_modules/mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==", + "license": "MIT", + "dependencies": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object/node_modules/for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmmirror.com/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/moment": { "version": "2.30.1", "resolved": "https://registry.npmmirror.com/moment/-/moment-2.30.1.tgz", @@ -1690,6 +1944,15 @@ "node": ">= 0.8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-to-regexp": { "version": "6.3.0", "resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz", @@ -1797,6 +2060,142 @@ "node": ">=18" } }, + "node_modules/puppeteer-extra": { + "version": "3.3.6", + "resolved": "https://registry.npmmirror.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz", + "integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==", + "license": "MIT", + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "deepmerge": "^4.2.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "@types/puppeteer": "*", + "puppeteer": "*", + "puppeteer-core": "*" + }, + "peerDependenciesMeta": { + "@types/puppeteer": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "puppeteer-core": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin": { + "version": "3.2.3", + "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz", + "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==", + "license": "MIT", + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "merge-deep": "^3.0.1" + }, + "engines": { + "node": ">=9.11.2" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-stealth": { + "version": "2.11.2", + "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz", + "integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-preferences": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-data-dir": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz", + "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "fs-extra": "^10.0.0", + "puppeteer-extra-plugin": "^3.2.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-preferences": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz", + "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "deepmerge": "^4.2.2", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-data-dir": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, "node_modules/qs": { "version": "6.15.0", "resolved": "https://registry.npmmirror.com/qs/-/qs-6.15.0.tgz", @@ -1863,6 +2262,22 @@ "integrity": "sha512-hMD7odLOt3LkTjcif8aRZqi/hybjpLNgSk5oF5FCowfCjok6LukpN2bDX7R5wDmbgBQFn7YoBxSagmtXHaJYJw==", "license": "MIT" }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1995,6 +2410,42 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmmirror.com/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==", + "license": "MIT", + "dependencies": { + "is-buffer": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.1.0.tgz", @@ -2291,6 +2742,15 @@ "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "license": "MIT" }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz", diff --git a/server/package.json b/server/package.json index 222dc40..8516042 100644 --- a/server/package.json +++ b/server/package.json @@ -14,6 +14,8 @@ "mysql2": "^3.11.0", "node-cron": "^3.0.3", "puppeteer": "^23.4.1", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2", "sequelize": "^6.37.3" } } diff --git a/server/services/puppeteer/puppeteer_runner.js b/server/services/puppeteer/puppeteer_runner.js new file mode 100644 index 0000000..87a1f78 --- /dev/null +++ b/server/services/puppeteer/puppeteer_runner.js @@ -0,0 +1,201 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; +import { get_app_config } from '../config/app_config.js'; +import { apply_page_stealth_defaults, get_stealth_puppeteer } from './puppeteer_stealth.js'; + +let browser_singleton = null; + +function get_action_timeout_ms() { + const cfg = get_app_config(); + return cfg.crawler.action_timeout_ms; +} + +function get_crx_src_path() { + const cfg = get_app_config(); + return cfg.crawler.crx_src_path; +} + +function get_extension_id_from_targets(targets) { + for (const target of targets) { + const url = target.url(); + if (!url) continue; + if (url.startsWith('chrome-extension://')) { + const match = url.match(/^chrome-extension:\/\/([^/]+)\//); + if (match && match[1]) return match[1]; + } + } + return null; +} + +async function wait_for_extension_id(browser, timeout_ms) { + const existing = get_extension_id_from_targets(browser.targets()); + if (existing) { + return existing; + } + + const target = await browser + .waitForTarget((t) => { + const url = t.url(); + return typeof url === 'string' && url.startsWith('chrome-extension://'); + }, { timeout: timeout_ms }) + .catch(() => null); + + if (!target) { + return null; + } + + return get_extension_id_from_targets([target]); +} + +function get_chrome_executable_path() { + const cfg = get_app_config(); + return path.resolve(cfg.crawler.chrome_executable_path); +} + +export async function get_or_create_browser() { + if (browser_singleton) { + return browser_singleton; + } + + const chrome_executable_path = get_chrome_executable_path(); + if (!fs.existsSync(chrome_executable_path)) { + throw new Error(`Chrome 不存在: ${chrome_executable_path}`); + } + + const raw_extension_path = path.resolve(get_crx_src_path()); + const manifest_path = path.resolve(raw_extension_path, 'manifest.json'); + if (!fs.existsSync(manifest_path)) { + throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`); + } + + const cfg = get_app_config(); + const extension_path = raw_extension_path.replace(/\\/g, '/'); + const headless = cfg.crawler.puppeteer_headless; + + const cfg2 = get_app_config(); + const pptr = cfg2.crawler.enable_stealth ? get_stealth_puppeteer(puppeteer) : puppeteer; + + browser_singleton = await pptr.launch({ + executablePath: chrome_executable_path, + headless, + args: [ + '--enable-extensions', + `--disable-extensions-except=${extension_path}`, + `--load-extension=${extension_path}`, + '--no-default-browser-check', + '--disable-popup-blocking', + '--disable-dev-shm-usage', + '--disable-features=ExtensionManifestV2Disabled,ExtensionManifestV2Unsupported', + '--enable-features=AllowLegacyMV2Extensions' + ] + }); + + return browser_singleton; +} + +export async function invoke_extension_action(action_name, action_payload) { + const cfg = get_app_config(); + const browser = await get_or_create_browser(); + + const started_at = Date.now(); + + const log_enabled = cfg.crawler.log_invoke_action; + if (log_enabled) { + // eslint-disable-next-line no-console + console.log('[invoke_extension_action] start', { + action_name, + has_payload: !!action_payload, + keys: action_payload && typeof action_payload === 'object' ? Object.keys(action_payload).slice(0, 20) : [] + }); + } + + let page = null; + try { + page = await browser.newPage(); + if (cfg.crawler.enable_stealth) { + await apply_page_stealth_defaults(page); + } + await page.goto('about:blank'); + + // 尝试先打开 chrome://extensions 触发扩展初始化（某些环境下扩展 target 不会立刻出现） + try { + await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' }); + } catch (err) { + // ignore + } + + const extension_id = await wait_for_extension_id(browser, 15000); + if (!extension_id) { + throw new Error( + '未找到扩展 extension_id：Chrome 未加载扩展（常见原因：MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless）' + ); + } + + const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`; + await page.goto(bridge_url, { waitUntil: 'domcontentloaded' }); + + const timeout_ms = get_action_timeout_ms(); + const action_res = await page.evaluate( + async (action, payload, timeout) => { + function with_timeout(promise, timeout_ms_inner) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner); + promise + .then((v) => { + clearTimeout(timer); + resolve(v); + }) + .catch((e) => { + clearTimeout(timer); + reject(e); + }); + }); + } + + if (!window.server_bridge_invoke) { + throw new Error('bridge 未注入 window.server_bridge_invoke'); + } + + return await with_timeout(window.server_bridge_invoke(action, payload), timeout); + }, + action_name, + action_payload || {}, + timeout_ms + ); + + if (log_enabled) { + // eslint-disable-next-line no-console + console.log('[invoke_extension_action] ok', { action_name, cost_ms: Date.now() - started_at }); + } + + return action_res; + } catch (err) { + if (log_enabled) { + // eslint-disable-next-line no-console + console.log('[invoke_extension_action] fail', { + action_name, + cost_ms: Date.now() - started_at, + error: (err && err.message) || String(err) + }); + } + throw err; + } finally { + if (page) { + try { + await page.close(); + } catch (err) { + // ignore + } + } + + if (cfg.crawler.auto_close_browser) { + try { + await browser.close(); + } catch (err) { + // ignore + } + browser_singleton = null; + } + } +} diff --git a/server/services/puppeteer/puppeteer_stealth.js b/server/services/puppeteer/puppeteer_stealth.js new file mode 100644 index 0000000..6a79de2 --- /dev/null +++ b/server/services/puppeteer/puppeteer_stealth.js @@ -0,0 +1,27 @@ +import puppeteer_extra from 'puppeteer-extra'; +import stealth_plugin from 'puppeteer-extra-plugin-stealth'; + +// 全局只注册一次插件 +let inited = false; + +export function get_stealth_puppeteer(puppeteer_core) { + if (!inited) { + puppeteer_extra.use(stealth_plugin()); + inited = true; + } + + // 复用 puppeteer 的 Chromium/Chrome 绑定（保持你现有的 executablePath 等能力） + puppeteer_extra.puppeteer = puppeteer_core; + return puppeteer_extra; +} + +export async function apply_page_stealth_defaults(page) { + // 这些属于通用的轻量“指纹一致性”设置，不会影响你现有业务 + await page.setViewport({ width: 1366, height: 768, deviceScaleFactor: 1 }); + await page.setUserAgent( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' + ); + await page.setExtraHTTPHeaders({ + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8' + }); +} diff --git a/server/services/puppeteer_runner.js b/server/services/puppeteer_runner.js deleted file mode 100644 index 5d9b223..0000000 --- a/server/services/puppeteer_runner.js +++ /dev/null @@ -1,148 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import puppeteer from 'puppeteer'; -import { get_app_config } from '../config/app_config.js'; - -let browser_singleton = null; - -function get_action_timeout_ms() { - const cfg = get_app_config(); - return cfg.crawler.action_timeout_ms; -} - -function get_crx_src_path() { - const cfg = get_app_config(); - return cfg.crawler.crx_src_path; -} - -function get_extension_id_from_targets(targets) { - for (const target of targets) { - const url = target.url(); - if (!url) continue; - if (url.startsWith('chrome-extension://')) { - const match = url.match(/^chrome-extension:\/\/([^/]+)\//); - if (match && match[1]) return match[1]; - } - } - return null; -} - -async function wait_for_extension_id(browser, timeout_ms) { - const existing = get_extension_id_from_targets(browser.targets()); - if (existing) { - return existing; - } - - const target = await browser - .waitForTarget((t) => { - const url = t.url(); - return typeof url === 'string' && url.startsWith('chrome-extension://'); - }, { timeout: timeout_ms }) - .catch(() => null); - - if (!target) { - return null; - } - - return get_extension_id_from_targets([target]); -} - -function get_chrome_executable_path() { - const cfg = get_app_config(); - return path.resolve(cfg.crawler.chrome_executable_path); -} - -export async function get_or_create_browser() { - if (browser_singleton) { - return browser_singleton; - } - - const chrome_executable_path = get_chrome_executable_path(); - if (!fs.existsSync(chrome_executable_path)) { - throw new Error(`Chrome 不存在: ${chrome_executable_path}`); - } - - const raw_extension_path = path.resolve(get_crx_src_path()); - const manifest_path = path.resolve(raw_extension_path, 'manifest.json'); - if (!fs.existsSync(manifest_path)) { - throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`); - } - - const cfg = get_app_config(); - const extension_path = raw_extension_path.replace(/\\/g, '/'); - const headless = cfg.crawler.puppeteer_headless; - const user_data_dir = path.resolve(process.cwd(), 'puppeteer_profile'); - - browser_singleton = await puppeteer.launch({ - executablePath: chrome_executable_path, - headless, - args: [ - `--user-data-dir=${user_data_dir}`, - '--enable-extensions', - `--disable-extensions-except=${extension_path}`, - `--load-extension=${extension_path}`, - '--no-default-browser-check', - '--disable-popup-blocking', - '--disable-dev-shm-usage' - ] - }); - - return browser_singleton; -} - -export async function invoke_extension_action(action_name, action_payload) { - const browser = await get_or_create_browser(); - - const page = await browser.newPage(); - await page.goto('about:blank'); - - // 尝试先打开 chrome://extensions 触发扩展初始化（某些环境下扩展 target 不会立刻出现） - try { - await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' }); - } catch (err) { - // ignore - } - - const extension_id = await wait_for_extension_id(browser, 15000); - if (!extension_id) { - await page.close(); - throw new Error( - '未找到扩展 extension_id：Chrome 未加载扩展（常见原因：MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless）' - ); - } - - const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`; - await page.goto(bridge_url, { waitUntil: 'domcontentloaded' }); - - const timeout_ms = get_action_timeout_ms(); - const action_res = await page.evaluate( - async (action, payload, timeout) => { - function with_timeout(promise, timeout_ms_inner) { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner); - promise - .then((v) => { - clearTimeout(timer); - resolve(v); - }) - .catch((e) => { - clearTimeout(timer); - reject(e); - }); - }); - } - - if (!window.server_bridge_invoke) { - throw new Error('bridge 未注入 window.server_bridge_invoke'); - } - - return await with_timeout(window.server_bridge_invoke(action, payload), timeout); - }, - action_name, - action_payload || {}, - timeout_ms - ); - - await page.close(); - return action_res; -} diff --git a/server/services/task_executor.js b/server/services/task_executor.js index fe3c7e7..fea9fd8 100644 --- a/server/services/task_executor.js +++ b/server/services/task_executor.js @@ -1,6 +1,6 @@ import { crawl_run_record } from '../models/index.js'; import { safe_json_stringify } from './json_utils.js'; -import { invoke_extension_action } from './puppeteer_runner.js'; +import { invoke_extension_action } from './puppeteer/puppeteer_runner.js'; import { persist_amazon_result } from './amazon_persist.js'; export async function execute_action_and_record(params) { @@ -13,18 +13,15 @@ export async function execute_action_and_record(params) { let error_message = null; try { + const result = await invoke_extension_action(action_name, action_payload || {}); - - console.log( 'invoke_extension_action-start', action_name, action_payload ); - const res_invoke = await invoke_extension_action(action_name, action_payload || {}); - console.log( 'invoke_extension_action-end', action_name, result ); ok = true; - result_payload = safe_json_stringify(res_invoke); + result_payload = safe_json_stringify(result); // 按 stage 自动入库（不影响原始 run_record 记录） - await persist_amazon_result(res_invoke.result); + await persist_amazon_result(result); - return res_invoke; + return result; } catch (err) { ok = false; error_message = (err && err.message) || String(err);