1
This commit is contained in:
张成
@@ -57,7 +57,8 @@ export function get_app_config() {
|
||||
puppeteer_headless: get_bool('PUPPETEER_HEADLESS', false),
|
||||
chrome_executable_path: (get_env('CHROME_EXECUTABLE_PATH') || '').trim() || path.resolve(__dirname, '../../chrome-win/chrome.exe'),
|
||||
log_invoke_action: get_bool('LOG_INVOKE_ACTION', true),
|
||||
auto_close_browser: get_bool('AUTO_CLOSE_BROWSER', true)
|
||||
auto_close_browser: get_bool('AUTO_CLOSE_BROWSER', true),
|
||||
enable_stealth: get_bool('ENABLE_STEALTH', true)
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import { get_app_config } from './app_config.js';
|
||||
|
||||
export function get_sequelize_options() {
|
||||
const cfg = get_app_config();
|
||||
console.log( 'get_sequelize_options', cfg.mysql );
|
||||
|
||||
return {
|
||||
host: cfg.mysql.host,
|
||||
|
||||
460
server/package-lock.json
generated
460
server/package-lock.json
generated
@@ -14,6 +14,8 @@
|
||||
"mysql2": "^3.11.0",
|
||||
"node-cron": "^3.0.3",
|
||||
"puppeteer": "^23.4.1",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"sequelize": "^6.37.3"
|
||||
}
|
||||
},
|
||||
@@ -182,6 +184,15 @@
|
||||
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
|
||||
"license": "Python-2.0"
|
||||
},
|
||||
"node_modules/arr-union": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/arr-union/-/arr-union-3.1.0.tgz",
|
||||
"integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ast-types": {
|
||||
"version": "0.13.4",
|
||||
"resolved": "https://registry.npmmirror.com/ast-types/-/ast-types-0.13.4.tgz",
|
||||
@@ -217,6 +228,12 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bare-events": {
|
||||
"version": "2.8.2",
|
||||
"resolved": "https://registry.npmmirror.com/bare-events/-/bare-events-2.8.2.tgz",
|
||||
@@ -333,6 +350,16 @@
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.12",
|
||||
"resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
||||
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz",
|
||||
@@ -453,6 +480,22 @@
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/clone-deep": {
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmmirror.com/clone-deep/-/clone-deep-0.2.4.tgz",
|
||||
"integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"for-own": "^0.1.3",
|
||||
"is-plain-object": "^2.0.1",
|
||||
"kind-of": "^3.0.2",
|
||||
"lazy-cache": "^1.0.3",
|
||||
"shallow-clone": "^0.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/co": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmmirror.com/co/-/co-4.6.0.tgz",
|
||||
@@ -497,6 +540,12 @@
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/content-disposition": {
|
||||
"version": "0.5.4",
|
||||
"resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz",
|
||||
@@ -595,6 +644,15 @@
|
||||
"integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/deepmerge": {
|
||||
"version": "4.3.1",
|
||||
"resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-4.3.1.tgz",
|
||||
"integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/degenerator": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/degenerator/-/degenerator-5.0.1.tgz",
|
||||
@@ -858,6 +916,27 @@
|
||||
"pend": "~1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/for-in": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/for-in/-/for-in-1.0.2.tgz",
|
||||
"integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/for-own": {
|
||||
"version": "0.1.5",
|
||||
"resolved": "https://registry.npmmirror.com/for-own/-/for-own-0.1.5.tgz",
|
||||
"integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"for-in": "^1.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fresh": {
|
||||
"version": "0.5.2",
|
||||
"resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz",
|
||||
@@ -867,6 +946,26 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/fs-extra": {
|
||||
"version": "10.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-10.1.0.tgz",
|
||||
"integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"graceful-fs": "^4.2.0",
|
||||
"jsonfile": "^6.0.1",
|
||||
"universalify": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/function-bind": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz",
|
||||
@@ -969,6 +1068,27 @@
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/glob": {
|
||||
"version": "7.2.3",
|
||||
"resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.3.tgz",
|
||||
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
|
||||
"deprecated": "Glob versions prior to v9 are no longer supported",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.1.1",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/gopd": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/gopd/-/gopd-1.2.0.tgz",
|
||||
@@ -981,6 +1101,12 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/graceful-fs": {
|
||||
"version": "4.2.11",
|
||||
"resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.11.tgz",
|
||||
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/has-symbols": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.1.0.tgz",
|
||||
@@ -1183,6 +1309,17 @@
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz",
|
||||
"integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
|
||||
"deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"once": "^1.3.0",
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz",
|
||||
@@ -1204,6 +1341,21 @@
|
||||
"integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-buffer": {
|
||||
"version": "1.1.6",
|
||||
"resolved": "https://registry.npmmirror.com/is-buffer/-/is-buffer-1.1.6.tgz",
|
||||
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-extendable": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmmirror.com/is-extendable/-/is-extendable-0.1.1.tgz",
|
||||
"integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-fullwidth-code-point": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||
@@ -1232,6 +1384,18 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/is-plain-object": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-2.0.4.tgz",
|
||||
"integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"isobject": "^3.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-property": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/is-property/-/is-property-1.0.2.tgz",
|
||||
@@ -1256,6 +1420,15 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/isobject": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/isobject/-/isobject-3.0.1.tgz",
|
||||
"integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmmirror.com/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
@@ -1280,6 +1453,18 @@
|
||||
"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/jsonfile": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.2.0.tgz",
|
||||
"integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"universalify": "^2.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"graceful-fs": "^4.1.6"
|
||||
}
|
||||
},
|
||||
"node_modules/keygrip": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/keygrip/-/keygrip-1.1.0.tgz",
|
||||
@@ -1293,6 +1478,18 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "3.2.2",
|
||||
"resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz",
|
||||
"integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-buffer": "^1.1.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/koa": {
|
||||
"version": "2.16.4",
|
||||
"resolved": "https://registry.npmmirror.com/koa/-/koa-2.16.4.tgz",
|
||||
@@ -1394,6 +1591,15 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/lazy-cache": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-1.0.4.tgz",
|
||||
"integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lines-and-columns": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmmirror.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
|
||||
@@ -1454,6 +1660,20 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/merge-deep": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/merge-deep/-/merge-deep-3.0.3.tgz",
|
||||
"integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"arr-union": "^3.1.0",
|
||||
"clone-deep": "^0.2.4",
|
||||
"kind-of": "^3.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/methods": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz",
|
||||
@@ -1484,12 +1704,46 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "3.1.5",
|
||||
"resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.5.tgz",
|
||||
"integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/mitt": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
|
||||
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/mixin-object": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/mixin-object/-/mixin-object-2.0.1.tgz",
|
||||
"integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"for-in": "^0.1.3",
|
||||
"is-extendable": "^0.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/mixin-object/node_modules/for-in": {
|
||||
"version": "0.1.8",
|
||||
"resolved": "https://registry.npmmirror.com/for-in/-/for-in-0.1.8.tgz",
|
||||
"integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/moment": {
|
||||
"version": "2.30.1",
|
||||
"resolved": "https://registry.npmmirror.com/moment/-/moment-2.30.1.tgz",
|
||||
@@ -1690,6 +1944,15 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/path-to-regexp": {
|
||||
"version": "6.3.0",
|
||||
"resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
|
||||
@@ -1797,6 +2060,142 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/puppeteer-extra": {
|
||||
"version": "3.3.6",
|
||||
"resolved": "https://registry.npmmirror.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz",
|
||||
"integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/debug": "^4.1.0",
|
||||
"debug": "^4.1.1",
|
||||
"deepmerge": "^4.2.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/puppeteer": "*",
|
||||
"puppeteer": "*",
|
||||
"puppeteer-core": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/puppeteer": {
|
||||
"optional": true
|
||||
},
|
||||
"puppeteer": {
|
||||
"optional": true
|
||||
},
|
||||
"puppeteer-core": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/puppeteer-extra-plugin": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz",
|
||||
"integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/debug": "^4.1.0",
|
||||
"debug": "^4.1.1",
|
||||
"merge-deep": "^3.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=9.11.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"playwright-extra": "*",
|
||||
"puppeteer-extra": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"playwright-extra": {
|
||||
"optional": true
|
||||
},
|
||||
"puppeteer-extra": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/puppeteer-extra-plugin-stealth": {
|
||||
"version": "2.11.2",
|
||||
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz",
|
||||
"integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "^4.1.1",
|
||||
"puppeteer-extra-plugin": "^3.2.3",
|
||||
"puppeteer-extra-plugin-user-preferences": "^2.4.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"playwright-extra": "*",
|
||||
"puppeteer-extra": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"playwright-extra": {
|
||||
"optional": true
|
||||
},
|
||||
"puppeteer-extra": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/puppeteer-extra-plugin-user-data-dir": {
|
||||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz",
|
||||
"integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "^4.1.1",
|
||||
"fs-extra": "^10.0.0",
|
||||
"puppeteer-extra-plugin": "^3.2.3",
|
||||
"rimraf": "^3.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"playwright-extra": "*",
|
||||
"puppeteer-extra": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"playwright-extra": {
|
||||
"optional": true
|
||||
},
|
||||
"puppeteer-extra": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/puppeteer-extra-plugin-user-preferences": {
|
||||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz",
|
||||
"integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "^4.1.1",
|
||||
"deepmerge": "^4.2.2",
|
||||
"puppeteer-extra-plugin": "^3.2.3",
|
||||
"puppeteer-extra-plugin-user-data-dir": "^2.4.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"playwright-extra": "*",
|
||||
"puppeteer-extra": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"playwright-extra": {
|
||||
"optional": true
|
||||
},
|
||||
"puppeteer-extra": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/qs": {
|
||||
"version": "6.15.0",
|
||||
"resolved": "https://registry.npmmirror.com/qs/-/qs-6.15.0.tgz",
|
||||
@@ -1863,6 +2262,22 @@
|
||||
"integrity": "sha512-hMD7odLOt3LkTjcif8aRZqi/hybjpLNgSk5oF5FCowfCjok6LukpN2bDX7R5wDmbgBQFn7YoBxSagmtXHaJYJw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz",
|
||||
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
|
||||
"deprecated": "Rimraf versions prior to v4 are no longer supported",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"glob": "^7.1.3"
|
||||
},
|
||||
"bin": {
|
||||
"rimraf": "bin.js"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
@@ -1995,6 +2410,42 @@
|
||||
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/shallow-clone": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmmirror.com/shallow-clone/-/shallow-clone-0.1.2.tgz",
|
||||
"integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-extendable": "^0.1.1",
|
||||
"kind-of": "^2.0.1",
|
||||
"lazy-cache": "^0.2.3",
|
||||
"mixin-object": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/shallow-clone/node_modules/kind-of": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-2.0.1.tgz",
|
||||
"integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-buffer": "^1.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/shallow-clone/node_modules/lazy-cache": {
|
||||
"version": "0.2.7",
|
||||
"resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-0.2.7.tgz",
|
||||
"integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/side-channel": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.1.0.tgz",
|
||||
@@ -2291,6 +2742,15 @@
|
||||
"integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/universalify": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.1.tgz",
|
||||
"integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/unpipe": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz",
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
"mysql2": "^3.11.0",
|
||||
"node-cron": "^3.0.3",
|
||||
"puppeteer": "^23.4.1",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"sequelize": "^6.37.3"
|
||||
}
|
||||
}
|
||||
|
||||
201
server/services/puppeteer/puppeteer_runner.js
Normal file
201
server/services/puppeteer/puppeteer_runner.js
Normal file
@@ -0,0 +1,201 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import puppeteer from 'puppeteer';
|
||||
import { get_app_config } from '../config/app_config.js';
|
||||
import { apply_page_stealth_defaults, get_stealth_puppeteer } from './puppeteer_stealth.js';
|
||||
|
||||
let browser_singleton = null;
|
||||
|
||||
function get_action_timeout_ms() {
|
||||
const cfg = get_app_config();
|
||||
return cfg.crawler.action_timeout_ms;
|
||||
}
|
||||
|
||||
function get_crx_src_path() {
|
||||
const cfg = get_app_config();
|
||||
return cfg.crawler.crx_src_path;
|
||||
}
|
||||
|
||||
function get_extension_id_from_targets(targets) {
|
||||
for (const target of targets) {
|
||||
const url = target.url();
|
||||
if (!url) continue;
|
||||
if (url.startsWith('chrome-extension://')) {
|
||||
const match = url.match(/^chrome-extension:\/\/([^/]+)\//);
|
||||
if (match && match[1]) return match[1];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function wait_for_extension_id(browser, timeout_ms) {
|
||||
const existing = get_extension_id_from_targets(browser.targets());
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const target = await browser
|
||||
.waitForTarget((t) => {
|
||||
const url = t.url();
|
||||
return typeof url === 'string' && url.startsWith('chrome-extension://');
|
||||
}, { timeout: timeout_ms })
|
||||
.catch(() => null);
|
||||
|
||||
if (!target) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return get_extension_id_from_targets([target]);
|
||||
}
|
||||
|
||||
function get_chrome_executable_path() {
|
||||
const cfg = get_app_config();
|
||||
return path.resolve(cfg.crawler.chrome_executable_path);
|
||||
}
|
||||
|
||||
export async function get_or_create_browser() {
|
||||
if (browser_singleton) {
|
||||
return browser_singleton;
|
||||
}
|
||||
|
||||
const chrome_executable_path = get_chrome_executable_path();
|
||||
if (!fs.existsSync(chrome_executable_path)) {
|
||||
throw new Error(`Chrome 不存在: ${chrome_executable_path}`);
|
||||
}
|
||||
|
||||
const raw_extension_path = path.resolve(get_crx_src_path());
|
||||
const manifest_path = path.resolve(raw_extension_path, 'manifest.json');
|
||||
if (!fs.existsSync(manifest_path)) {
|
||||
throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`);
|
||||
}
|
||||
|
||||
const cfg = get_app_config();
|
||||
const extension_path = raw_extension_path.replace(/\\/g, '/');
|
||||
const headless = cfg.crawler.puppeteer_headless;
|
||||
|
||||
const cfg2 = get_app_config();
|
||||
const pptr = cfg2.crawler.enable_stealth ? get_stealth_puppeteer(puppeteer) : puppeteer;
|
||||
|
||||
browser_singleton = await pptr.launch({
|
||||
executablePath: chrome_executable_path,
|
||||
headless,
|
||||
args: [
|
||||
'--enable-extensions',
|
||||
`--disable-extensions-except=${extension_path}`,
|
||||
`--load-extension=${extension_path}`,
|
||||
'--no-default-browser-check',
|
||||
'--disable-popup-blocking',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-features=ExtensionManifestV2Disabled,ExtensionManifestV2Unsupported',
|
||||
'--enable-features=AllowLegacyMV2Extensions'
|
||||
]
|
||||
});
|
||||
|
||||
return browser_singleton;
|
||||
}
|
||||
|
||||
export async function invoke_extension_action(action_name, action_payload) {
|
||||
const cfg = get_app_config();
|
||||
const browser = await get_or_create_browser();
|
||||
|
||||
const started_at = Date.now();
|
||||
|
||||
const log_enabled = cfg.crawler.log_invoke_action;
|
||||
if (log_enabled) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[invoke_extension_action] start', {
|
||||
action_name,
|
||||
has_payload: !!action_payload,
|
||||
keys: action_payload && typeof action_payload === 'object' ? Object.keys(action_payload).slice(0, 20) : []
|
||||
});
|
||||
}
|
||||
|
||||
let page = null;
|
||||
try {
|
||||
page = await browser.newPage();
|
||||
if (cfg.crawler.enable_stealth) {
|
||||
await apply_page_stealth_defaults(page);
|
||||
}
|
||||
await page.goto('about:blank');
|
||||
|
||||
// 尝试先打开 chrome://extensions 触发扩展初始化（某些环境下扩展 target 不会立刻出现）
|
||||
try {
|
||||
await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' });
|
||||
} catch (err) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
const extension_id = await wait_for_extension_id(browser, 15000);
|
||||
if (!extension_id) {
|
||||
throw new Error(
|
||||
'未找到扩展 extension_id：Chrome 未加载扩展（常见原因：MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless）'
|
||||
);
|
||||
}
|
||||
|
||||
const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`;
|
||||
await page.goto(bridge_url, { waitUntil: 'domcontentloaded' });
|
||||
|
||||
const timeout_ms = get_action_timeout_ms();
|
||||
const action_res = await page.evaluate(
|
||||
async (action, payload, timeout) => {
|
||||
function with_timeout(promise, timeout_ms_inner) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner);
|
||||
promise
|
||||
.then((v) => {
|
||||
clearTimeout(timer);
|
||||
resolve(v);
|
||||
})
|
||||
.catch((e) => {
|
||||
clearTimeout(timer);
|
||||
reject(e);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (!window.server_bridge_invoke) {
|
||||
throw new Error('bridge 未注入 window.server_bridge_invoke');
|
||||
}
|
||||
|
||||
return await with_timeout(window.server_bridge_invoke(action, payload), timeout);
|
||||
},
|
||||
action_name,
|
||||
action_payload || {},
|
||||
timeout_ms
|
||||
);
|
||||
|
||||
if (log_enabled) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[invoke_extension_action] ok', { action_name, cost_ms: Date.now() - started_at });
|
||||
}
|
||||
|
||||
return action_res;
|
||||
} catch (err) {
|
||||
if (log_enabled) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[invoke_extension_action] fail', {
|
||||
action_name,
|
||||
cost_ms: Date.now() - started_at,
|
||||
error: (err && err.message) || String(err)
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
if (page) {
|
||||
try {
|
||||
await page.close();
|
||||
} catch (err) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
if (cfg.crawler.auto_close_browser) {
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (err) {
|
||||
// ignore
|
||||
}
|
||||
browser_singleton = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
27
server/services/puppeteer/puppeteer_stealth.js
Normal file
27
server/services/puppeteer/puppeteer_stealth.js
Normal file
@@ -0,0 +1,27 @@
|
||||
import puppeteer_extra from 'puppeteer-extra';
|
||||
import stealth_plugin from 'puppeteer-extra-plugin-stealth';
|
||||
|
||||
// 全局只注册一次插件
|
||||
let inited = false;
|
||||
|
||||
export function get_stealth_puppeteer(puppeteer_core) {
|
||||
if (!inited) {
|
||||
puppeteer_extra.use(stealth_plugin());
|
||||
inited = true;
|
||||
}
|
||||
|
||||
// 复用 puppeteer 的 Chromium/Chrome 绑定（保持你现有的 executablePath 等能力）
|
||||
puppeteer_extra.puppeteer = puppeteer_core;
|
||||
return puppeteer_extra;
|
||||
}
|
||||
|
||||
export async function apply_page_stealth_defaults(page) {
|
||||
// 这些属于通用的轻量“指纹一致性”设置，不会影响你现有业务
|
||||
await page.setViewport({ width: 1366, height: 768, deviceScaleFactor: 1 });
|
||||
await page.setUserAgent(
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
|
||||
);
|
||||
await page.setExtraHTTPHeaders({
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
|
||||
});
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import puppeteer from 'puppeteer';
|
||||
import { get_app_config } from '../config/app_config.js';
|
||||
|
||||
let browser_singleton = null;
|
||||
|
||||
function get_action_timeout_ms() {
|
||||
const cfg = get_app_config();
|
||||
return cfg.crawler.action_timeout_ms;
|
||||
}
|
||||
|
||||
function get_crx_src_path() {
|
||||
const cfg = get_app_config();
|
||||
return cfg.crawler.crx_src_path;
|
||||
}
|
||||
|
||||
function get_extension_id_from_targets(targets) {
|
||||
for (const target of targets) {
|
||||
const url = target.url();
|
||||
if (!url) continue;
|
||||
if (url.startsWith('chrome-extension://')) {
|
||||
const match = url.match(/^chrome-extension:\/\/([^/]+)\//);
|
||||
if (match && match[1]) return match[1];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function wait_for_extension_id(browser, timeout_ms) {
|
||||
const existing = get_extension_id_from_targets(browser.targets());
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const target = await browser
|
||||
.waitForTarget((t) => {
|
||||
const url = t.url();
|
||||
return typeof url === 'string' && url.startsWith('chrome-extension://');
|
||||
}, { timeout: timeout_ms })
|
||||
.catch(() => null);
|
||||
|
||||
if (!target) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return get_extension_id_from_targets([target]);
|
||||
}
|
||||
|
||||
function get_chrome_executable_path() {
|
||||
const cfg = get_app_config();
|
||||
return path.resolve(cfg.crawler.chrome_executable_path);
|
||||
}
|
||||
|
||||
export async function get_or_create_browser() {
|
||||
if (browser_singleton) {
|
||||
return browser_singleton;
|
||||
}
|
||||
|
||||
const chrome_executable_path = get_chrome_executable_path();
|
||||
if (!fs.existsSync(chrome_executable_path)) {
|
||||
throw new Error(`Chrome 不存在: ${chrome_executable_path}`);
|
||||
}
|
||||
|
||||
const raw_extension_path = path.resolve(get_crx_src_path());
|
||||
const manifest_path = path.resolve(raw_extension_path, 'manifest.json');
|
||||
if (!fs.existsSync(manifest_path)) {
|
||||
throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`);
|
||||
}
|
||||
|
||||
const cfg = get_app_config();
|
||||
const extension_path = raw_extension_path.replace(/\\/g, '/');
|
||||
const headless = cfg.crawler.puppeteer_headless;
|
||||
const user_data_dir = path.resolve(process.cwd(), 'puppeteer_profile');
|
||||
|
||||
browser_singleton = await puppeteer.launch({
|
||||
executablePath: chrome_executable_path,
|
||||
headless,
|
||||
args: [
|
||||
`--user-data-dir=${user_data_dir}`,
|
||||
'--enable-extensions',
|
||||
`--disable-extensions-except=${extension_path}`,
|
||||
`--load-extension=${extension_path}`,
|
||||
'--no-default-browser-check',
|
||||
'--disable-popup-blocking',
|
||||
'--disable-dev-shm-usage'
|
||||
]
|
||||
});
|
||||
|
||||
return browser_singleton;
|
||||
}
|
||||
|
||||
export async function invoke_extension_action(action_name, action_payload) {
|
||||
const browser = await get_or_create_browser();
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.goto('about:blank');
|
||||
|
||||
// 尝试先打开 chrome://extensions 触发扩展初始化（某些环境下扩展 target 不会立刻出现）
|
||||
try {
|
||||
await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' });
|
||||
} catch (err) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
const extension_id = await wait_for_extension_id(browser, 15000);
|
||||
if (!extension_id) {
|
||||
await page.close();
|
||||
throw new Error(
|
||||
'未找到扩展 extension_id：Chrome 未加载扩展（常见原因：MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless）'
|
||||
);
|
||||
}
|
||||
|
||||
const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`;
|
||||
await page.goto(bridge_url, { waitUntil: 'domcontentloaded' });
|
||||
|
||||
const timeout_ms = get_action_timeout_ms();
|
||||
const action_res = await page.evaluate(
|
||||
async (action, payload, timeout) => {
|
||||
function with_timeout(promise, timeout_ms_inner) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner);
|
||||
promise
|
||||
.then((v) => {
|
||||
clearTimeout(timer);
|
||||
resolve(v);
|
||||
})
|
||||
.catch((e) => {
|
||||
clearTimeout(timer);
|
||||
reject(e);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (!window.server_bridge_invoke) {
|
||||
throw new Error('bridge 未注入 window.server_bridge_invoke');
|
||||
}
|
||||
|
||||
return await with_timeout(window.server_bridge_invoke(action, payload), timeout);
|
||||
},
|
||||
action_name,
|
||||
action_payload || {},
|
||||
timeout_ms
|
||||
);
|
||||
|
||||
await page.close();
|
||||
return action_res;
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { crawl_run_record } from '../models/index.js';
|
||||
import { safe_json_stringify } from './json_utils.js';
|
||||
import { invoke_extension_action } from './puppeteer_runner.js';
|
||||
import { invoke_extension_action } from './puppeteer/puppeteer_runner.js';
|
||||
import { persist_amazon_result } from './amazon_persist.js';
|
||||
|
||||
export async function execute_action_and_record(params) {
|
||||
@@ -13,18 +13,15 @@ export async function execute_action_and_record(params) {
|
||||
let error_message = null;
|
||||
|
||||
try {
|
||||
const result = await invoke_extension_action(action_name, action_payload || {});
|
||||
|
||||
|
||||
console.log( 'invoke_extension_action-start', action_name, action_payload );
|
||||
const res_invoke = await invoke_extension_action(action_name, action_payload || {});
|
||||
console.log( 'invoke_extension_action-end', action_name, result );
|
||||
ok = true;
|
||||
result_payload = safe_json_stringify(res_invoke);
|
||||
result_payload = safe_json_stringify(result);
|
||||
|
||||
// 按 stage 自动入库（不影响原始 run_record 记录）
|
||||
await persist_amazon_result(res_invoke.result);
|
||||
await persist_amazon_result(result);
|
||||
|
||||
return res_invoke;
|
||||
return result;
|
||||
} catch (err) {
|
||||
ok = false;
|
||||
error_message = (err && err.message) || String(err);
|
||||
|
||||
Reference in New Issue
Block a user