1

server/config/app_config.js

@@ -57,7 +57,8 @@ export function get_app_config() {
       puppeteer_headless: get_bool('PUPPETEER_HEADLESS', false),
       chrome_executable_path: (get_env('CHROME_EXECUTABLE_PATH') || '').trim() || path.resolve(__dirname, '../../chrome-win/chrome.exe'),
       log_invoke_action: get_bool('LOG_INVOKE_ACTION', true),
-      auto_close_browser: get_bool('AUTO_CLOSE_BROWSER', true)
+      auto_close_browser: get_bool('AUTO_CLOSE_BROWSER', true),
+      enable_stealth: get_bool('ENABLE_STEALTH', true)
     }
   };
 

server/config/database.js

@@ -2,7 +2,6 @@ import { get_app_config } from './app_config.js';
 
 export function get_sequelize_options() {
   const cfg = get_app_config();
-  console.log( 'get_sequelize_options', cfg.mysql );
 
   return {
     host: cfg.mysql.host,

server/package-lock.json

@@ -14,6 +14,8 @@
         "mysql2": "^3.11.0",
         "node-cron": "^3.0.3",
         "puppeteer": "^23.4.1",
+        "puppeteer-extra": "^3.3.6",
+        "puppeteer-extra-plugin-stealth": "^2.11.2",
         "sequelize": "^6.37.3"
       }
     },
@@ -182,6 +184,15 @@
       "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
       "license": "Python-2.0"
     },
+    "node_modules/arr-union": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmmirror.com/arr-union/-/arr-union-3.1.0.tgz",
+      "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/ast-types": {
       "version": "0.13.4",
       "resolved": "https://registry.npmmirror.com/ast-types/-/ast-types-0.13.4.tgz",
@@ -217,6 +228,12 @@
         }
       }
     },
+    "node_modules/balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+      "license": "MIT"
+    },
     "node_modules/bare-events": {
       "version": "2.8.2",
       "resolved": "https://registry.npmmirror.com/bare-events/-/bare-events-2.8.2.tgz",
@@ -333,6 +350,16 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/brace-expansion": {
+      "version": "1.1.12",
+      "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.12.tgz",
+      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
+      "license": "MIT",
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
     "node_modules/buffer": {
       "version": "5.7.1",
       "resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz",
@@ -453,6 +480,22 @@
         "node": ">=12"
       }
     },
+    "node_modules/clone-deep": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmmirror.com/clone-deep/-/clone-deep-0.2.4.tgz",
+      "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==",
+      "license": "MIT",
+      "dependencies": {
+        "for-own": "^0.1.3",
+        "is-plain-object": "^2.0.1",
+        "kind-of": "^3.0.2",
+        "lazy-cache": "^1.0.3",
+        "shallow-clone": "^0.1.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmmirror.com/co/-/co-4.6.0.tgz",
@@ -497,6 +540,12 @@
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "license": "MIT"
     },
+    "node_modules/concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
+      "license": "MIT"
+    },
     "node_modules/content-disposition": {
       "version": "0.5.4",
       "resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -595,6 +644,15 @@
       "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==",
       "license": "MIT"
     },
+    "node_modules/deepmerge": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-4.3.1.tgz",
+      "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/degenerator": {
       "version": "5.0.1",
       "resolved": "https://registry.npmmirror.com/degenerator/-/degenerator-5.0.1.tgz",
@@ -858,6 +916,27 @@
         "pend": "~1.2.0"
       }
     },
+    "node_modules/for-in": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmmirror.com/for-in/-/for-in-1.0.2.tgz",
+      "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/for-own": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmmirror.com/for-own/-/for-own-0.1.5.tgz",
+      "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==",
+      "license": "MIT",
+      "dependencies": {
+        "for-in": "^1.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/fresh": {
       "version": "0.5.2",
       "resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz",
@@ -867,6 +946,26 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/fs-extra": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-10.1.0.tgz",
+      "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
+      "license": "MIT",
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+      "license": "ISC"
+    },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz",
@@ -969,6 +1068,27 @@
         "node": ">= 14"
       }
     },
+    "node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "deprecated": "Glob versions prior to v9 are no longer supported",
+      "license": "ISC",
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/gopd": {
       "version": "1.2.0",
       "resolved": "https://registry.npmmirror.com/gopd/-/gopd-1.2.0.tgz",
@@ -981,6 +1101,12 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/graceful-fs": {
+      "version": "4.2.11",
+      "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.11.tgz",
+      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
+      "license": "ISC"
+    },
     "node_modules/has-symbols": {
       "version": "1.1.0",
       "resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.1.0.tgz",
@@ -1183,6 +1309,17 @@
       ],
       "license": "MIT"
     },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+      "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
+      "license": "ISC",
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz",
@@ -1204,6 +1341,21 @@
       "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
       "license": "MIT"
     },
+    "node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmmirror.com/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+      "license": "MIT"
+    },
+    "node_modules/is-extendable": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmmirror.com/is-extendable/-/is-extendable-0.1.1.tgz",
+      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -1232,6 +1384,18 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "license": "MIT",
+      "dependencies": {
+        "isobject": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-property": {
       "version": "1.0.2",
       "resolved": "https://registry.npmmirror.com/is-property/-/is-property-1.0.2.tgz",
@@ -1256,6 +1420,15 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmmirror.com/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmmirror.com/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -1280,6 +1453,18 @@
       "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
       "license": "MIT"
     },
+    "node_modules/jsonfile": {
+      "version": "6.2.0",
+      "resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.2.0.tgz",
+      "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==",
+      "license": "MIT",
+      "dependencies": {
+        "universalify": "^2.0.0"
+      },
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "node_modules/keygrip": {
       "version": "1.1.0",
       "resolved": "https://registry.npmmirror.com/keygrip/-/keygrip-1.1.0.tgz",
@@ -1293,6 +1478,18 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/kind-of": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz",
+      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+      "license": "MIT",
+      "dependencies": {
+        "is-buffer": "^1.1.5"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/koa": {
       "version": "2.16.4",
       "resolved": "https://registry.npmmirror.com/koa/-/koa-2.16.4.tgz",
@@ -1394,6 +1591,15 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/lazy-cache": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-1.0.4.tgz",
+      "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmmirror.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@@ -1454,6 +1660,20 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/merge-deep": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmmirror.com/merge-deep/-/merge-deep-3.0.3.tgz",
+      "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
+      "license": "MIT",
+      "dependencies": {
+        "arr-union": "^3.1.0",
+        "clone-deep": "^0.2.4",
+        "kind-of": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/methods": {
       "version": "1.1.2",
       "resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz",
@@ -1484,12 +1704,46 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/minimatch": {
+      "version": "3.1.5",
+      "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.5.tgz",
+      "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/mitt": {
       "version": "3.0.1",
       "resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
       "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
       "license": "MIT"
     },
+    "node_modules/mixin-object": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmmirror.com/mixin-object/-/mixin-object-2.0.1.tgz",
+      "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==",
+      "license": "MIT",
+      "dependencies": {
+        "for-in": "^0.1.3",
+        "is-extendable": "^0.1.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/mixin-object/node_modules/for-in": {
+      "version": "0.1.8",
+      "resolved": "https://registry.npmmirror.com/for-in/-/for-in-0.1.8.tgz",
+      "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/moment": {
       "version": "2.30.1",
       "resolved": "https://registry.npmmirror.com/moment/-/moment-2.30.1.tgz",
@@ -1690,6 +1944,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/path-to-regexp": {
       "version": "6.3.0",
       "resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
@@ -1797,6 +2060,142 @@
         "node": ">=18"
       }
     },
+    "node_modules/puppeteer-extra": {
+      "version": "3.3.6",
+      "resolved": "https://registry.npmmirror.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz",
+      "integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/debug": "^4.1.0",
+        "debug": "^4.1.1",
+        "deepmerge": "^4.2.2"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "@types/puppeteer": "*",
+        "puppeteer": "*",
+        "puppeteer-core": "*"
+      },
+      "peerDependenciesMeta": {
+        "@types/puppeteer": {
+          "optional": true
+        },
+        "puppeteer": {
+          "optional": true
+        },
+        "puppeteer-core": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin": {
+      "version": "3.2.3",
+      "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz",
+      "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/debug": "^4.1.0",
+        "debug": "^4.1.1",
+        "merge-deep": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=9.11.2"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin-stealth": {
+      "version": "2.11.2",
+      "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz",
+      "integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "^4.1.1",
+        "puppeteer-extra-plugin": "^3.2.3",
+        "puppeteer-extra-plugin-user-preferences": "^2.4.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin-user-data-dir": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz",
+      "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "^4.1.1",
+        "fs-extra": "^10.0.0",
+        "puppeteer-extra-plugin": "^3.2.3",
+        "rimraf": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin-user-preferences": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz",
+      "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "^4.1.1",
+        "deepmerge": "^4.2.2",
+        "puppeteer-extra-plugin": "^3.2.3",
+        "puppeteer-extra-plugin-user-data-dir": "^2.4.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/qs": {
       "version": "6.15.0",
       "resolved": "https://registry.npmmirror.com/qs/-/qs-6.15.0.tgz",
@@ -1863,6 +2262,22 @@
       "integrity": "sha512-hMD7odLOt3LkTjcif8aRZqi/hybjpLNgSk5oF5FCowfCjok6LukpN2bDX7R5wDmbgBQFn7YoBxSagmtXHaJYJw==",
       "license": "MIT"
     },
+    "node_modules/rimraf": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz",
+      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "deprecated": "Rimraf versions prior to v4 are no longer supported",
+      "license": "ISC",
+      "dependencies": {
+        "glob": "^7.1.3"
+      },
+      "bin": {
+        "rimraf": "bin.js"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -1995,6 +2410,42 @@
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
       "license": "ISC"
     },
+    "node_modules/shallow-clone": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmmirror.com/shallow-clone/-/shallow-clone-0.1.2.tgz",
+      "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==",
+      "license": "MIT",
+      "dependencies": {
+        "is-extendable": "^0.1.1",
+        "kind-of": "^2.0.1",
+        "lazy-cache": "^0.2.3",
+        "mixin-object": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/shallow-clone/node_modules/kind-of": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-2.0.1.tgz",
+      "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==",
+      "license": "MIT",
+      "dependencies": {
+        "is-buffer": "^1.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/shallow-clone/node_modules/lazy-cache": {
+      "version": "0.2.7",
+      "resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-0.2.7.tgz",
+      "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/side-channel": {
       "version": "1.1.0",
       "resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.1.0.tgz",
@@ -2291,6 +2742,15 @@
       "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
       "license": "MIT"
     },
+    "node_modules/universalify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.1.tgz",
+      "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 10.0.0"
+      }
+    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz",

server/package.json

@@ -14,6 +14,8 @@
     "mysql2": "^3.11.0",
     "node-cron": "^3.0.3",
     "puppeteer": "^23.4.1",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
     "sequelize": "^6.37.3"
   }
 }

server/services/puppeteer/puppeteer_runner.js

@@ -0,0 +1,201 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import puppeteer from 'puppeteer';
+import { get_app_config } from '../config/app_config.js';
+import { apply_page_stealth_defaults, get_stealth_puppeteer } from './puppeteer_stealth.js';
+
+let browser_singleton = null;
+
+function get_action_timeout_ms() {
+  const cfg = get_app_config();
+  return cfg.crawler.action_timeout_ms;
+}
+
+function get_crx_src_path() {
+  const cfg = get_app_config();
+  return cfg.crawler.crx_src_path;
+}
+
+function get_extension_id_from_targets(targets) {
+  for (const target of targets) {
+    const url = target.url();
+    if (!url) continue;
+    if (url.startsWith('chrome-extension://')) {
+      const match = url.match(/^chrome-extension:\/\/([^/]+)\//);
+      if (match && match[1]) return match[1];
+    }
+  }
+  return null;
+}
+
+async function wait_for_extension_id(browser, timeout_ms) {
+  const existing = get_extension_id_from_targets(browser.targets());
+  if (existing) {
+    return existing;
+  }
+
+  const target = await browser
+    .waitForTarget((t) => {
+      const url = t.url();
+      return typeof url === 'string' && url.startsWith('chrome-extension://');
+    }, { timeout: timeout_ms })
+    .catch(() => null);
+
+  if (!target) {
+    return null;
+  }
+
+  return get_extension_id_from_targets([target]);
+}
+
+function get_chrome_executable_path() {
+  const cfg = get_app_config();
+  return path.resolve(cfg.crawler.chrome_executable_path);
+}
+
+export async function get_or_create_browser() {
+  if (browser_singleton) {
+    return browser_singleton;
+  }
+
+  const chrome_executable_path = get_chrome_executable_path();
+  if (!fs.existsSync(chrome_executable_path)) {
+    throw new Error(`Chrome 不存在: ${chrome_executable_path}`);
+  }
+
+  const raw_extension_path = path.resolve(get_crx_src_path());
+  const manifest_path = path.resolve(raw_extension_path, 'manifest.json');
+  if (!fs.existsSync(manifest_path)) {
+    throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`);
+  }
+
+  const cfg = get_app_config();
+  const extension_path = raw_extension_path.replace(/\\/g, '/');
+  const headless = cfg.crawler.puppeteer_headless;
+
+  const cfg2 = get_app_config();
+  const pptr = cfg2.crawler.enable_stealth ? get_stealth_puppeteer(puppeteer) : puppeteer;
+
+  browser_singleton = await pptr.launch({
+    executablePath: chrome_executable_path,
+    headless,
+    args: [
+      '--enable-extensions',
+      `--disable-extensions-except=${extension_path}`,
+      `--load-extension=${extension_path}`,
+      '--no-default-browser-check',
+      '--disable-popup-blocking',
+      '--disable-dev-shm-usage',
+      '--disable-features=ExtensionManifestV2Disabled,ExtensionManifestV2Unsupported',
+      '--enable-features=AllowLegacyMV2Extensions'
+    ]
+  });
+
+  return browser_singleton;
+}
+
+export async function invoke_extension_action(action_name, action_payload) {
+  const cfg = get_app_config();
+  const browser = await get_or_create_browser();
+
+  const started_at = Date.now();
+
+  const log_enabled = cfg.crawler.log_invoke_action;
+  if (log_enabled) {
+    // eslint-disable-next-line no-console
+    console.log('[invoke_extension_action] start', {
+      action_name,
+      has_payload: !!action_payload,
+      keys: action_payload && typeof action_payload === 'object' ? Object.keys(action_payload).slice(0, 20) : []
+    });
+  }
+
+  let page = null;
+  try {
+    page = await browser.newPage();
+    if (cfg.crawler.enable_stealth) {
+      await apply_page_stealth_defaults(page);
+    }
+    await page.goto('about:blank');
+
+    // 尝试先打开 chrome://extensions 触发扩展初始化（某些环境下扩展 target 不会立刻出现）
+    try {
+      await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' });
+    } catch (err) {
+      // ignore
+    }
+
+    const extension_id = await wait_for_extension_id(browser, 15000);
+    if (!extension_id) {
+      throw new Error(
+        '未找到扩展 extension_id：Chrome 未加载扩展（常见原因：MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless）'
+      );
+    }
+
+    const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`;
+    await page.goto(bridge_url, { waitUntil: 'domcontentloaded' });
+
+    const timeout_ms = get_action_timeout_ms();
+    const action_res = await page.evaluate(
+      async (action, payload, timeout) => {
+        function with_timeout(promise, timeout_ms_inner) {
+          return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner);
+            promise
+              .then((v) => {
+                clearTimeout(timer);
+                resolve(v);
+              })
+              .catch((e) => {
+                clearTimeout(timer);
+                reject(e);
+              });
+          });
+        }
+
+        if (!window.server_bridge_invoke) {
+          throw new Error('bridge 未注入 window.server_bridge_invoke');
+        }
+
+        return await with_timeout(window.server_bridge_invoke(action, payload), timeout);
+      },
+      action_name,
+      action_payload || {},
+      timeout_ms
+    );
+
+    if (log_enabled) {
+      // eslint-disable-next-line no-console
+      console.log('[invoke_extension_action] ok', { action_name, cost_ms: Date.now() - started_at });
+    }
+
+    return action_res;
+  } catch (err) {
+    if (log_enabled) {
+      // eslint-disable-next-line no-console
+      console.log('[invoke_extension_action] fail', {
+        action_name,
+        cost_ms: Date.now() - started_at,
+        error: (err && err.message) || String(err)
+      });
+    }
+    throw err;
+  } finally {
+    if (page) {
+      try {
+        await page.close();
+      } catch (err) {
+        // ignore
+      }
+    }
+
+    if (cfg.crawler.auto_close_browser) {
+      try {
+        await browser.close();
+      } catch (err) {
+        // ignore
+      }
+      browser_singleton = null;
+    }
+  }
+}

server/services/puppeteer/puppeteer_stealth.js

@@ -0,0 +1,27 @@
+import puppeteer_extra from 'puppeteer-extra';
+import stealth_plugin from 'puppeteer-extra-plugin-stealth';
+
+// 全局只注册一次插件
+let inited = false;
+
+export function get_stealth_puppeteer(puppeteer_core) {
+  if (!inited) {
+    puppeteer_extra.use(stealth_plugin());
+    inited = true;
+  }
+
+  // 复用 puppeteer 的 Chromium/Chrome 绑定（保持你现有的 executablePath 等能力）
+  puppeteer_extra.puppeteer = puppeteer_core;
+  return puppeteer_extra;
+}
+
+export async function apply_page_stealth_defaults(page) {
+  // 这些属于通用的轻量“指纹一致性”设置，不会影响你现有业务
+  await page.setViewport({ width: 1366, height: 768, deviceScaleFactor: 1 });
+  await page.setUserAgent(
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+  );
+  await page.setExtraHTTPHeaders({
+    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
+  });
+}

server/services/puppeteer_runner.js

@@ -1,148 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import puppeteer from 'puppeteer';
-import { get_app_config } from '../config/app_config.js';
-
-let browser_singleton = null;
-
-function get_action_timeout_ms() {
-  const cfg = get_app_config();
-  return cfg.crawler.action_timeout_ms;
-}
-
-function get_crx_src_path() {
-  const cfg = get_app_config();
-  return cfg.crawler.crx_src_path;
-}
-
-function get_extension_id_from_targets(targets) {
-  for (const target of targets) {
-    const url = target.url();
-    if (!url) continue;
-    if (url.startsWith('chrome-extension://')) {
-      const match = url.match(/^chrome-extension:\/\/([^/]+)\//);
-      if (match && match[1]) return match[1];
-    }
-  }
-  return null;
-}
-
-async function wait_for_extension_id(browser, timeout_ms) {
-  const existing = get_extension_id_from_targets(browser.targets());
-  if (existing) {
-    return existing;
-  }
-
-  const target = await browser
-    .waitForTarget((t) => {
-      const url = t.url();
-      return typeof url === 'string' && url.startsWith('chrome-extension://');
-    }, { timeout: timeout_ms })
-    .catch(() => null);
-
-  if (!target) {
-    return null;
-  }
-
-  return get_extension_id_from_targets([target]);
-}
-
-function get_chrome_executable_path() {
-  const cfg = get_app_config();
-  return path.resolve(cfg.crawler.chrome_executable_path);
-}
-
-export async function get_or_create_browser() {
-  if (browser_singleton) {
-    return browser_singleton;
-  }
-
-  const chrome_executable_path = get_chrome_executable_path();
-  if (!fs.existsSync(chrome_executable_path)) {
-    throw new Error(`Chrome 不存在: ${chrome_executable_path}`);
-  }
-
-  const raw_extension_path = path.resolve(get_crx_src_path());
-  const manifest_path = path.resolve(raw_extension_path, 'manifest.json');
-  if (!fs.existsSync(manifest_path)) {
-    throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`);
-  }
-
-  const cfg = get_app_config();
-  const extension_path = raw_extension_path.replace(/\\/g, '/');
-  const headless = cfg.crawler.puppeteer_headless;
-  const user_data_dir = path.resolve(process.cwd(), 'puppeteer_profile');
-
-  browser_singleton = await puppeteer.launch({
-    executablePath: chrome_executable_path,
-    headless,
-    args: [
-      `--user-data-dir=${user_data_dir}`,
-      '--enable-extensions',
-      `--disable-extensions-except=${extension_path}`,
-      `--load-extension=${extension_path}`,
-      '--no-default-browser-check',
-      '--disable-popup-blocking',
-      '--disable-dev-shm-usage'
-    ]
-  });
-
-  return browser_singleton;
-}
-
-export async function invoke_extension_action(action_name, action_payload) {
-  const browser = await get_or_create_browser();
-
-  const page = await browser.newPage();
-  await page.goto('about:blank');
-
-  // 尝试先打开 chrome://extensions 触发扩展初始化（某些环境下扩展 target 不会立刻出现）
-  try {
-    await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' });
-  } catch (err) {
-    // ignore
-  }
-
-  const extension_id = await wait_for_extension_id(browser, 15000);
-  if (!extension_id) {
-    await page.close();
-    throw new Error(
-      '未找到扩展 extension_id：Chrome 未加载扩展（常见原因：MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless）'
-    );
-  }
-
-  const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`;
-  await page.goto(bridge_url, { waitUntil: 'domcontentloaded' });
-
-  const timeout_ms = get_action_timeout_ms();
-  const action_res = await page.evaluate(
-    async (action, payload, timeout) => {
-      function with_timeout(promise, timeout_ms_inner) {
-        return new Promise((resolve, reject) => {
-          const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner);
-          promise
-            .then((v) => {
-              clearTimeout(timer);
-              resolve(v);
-            })
-            .catch((e) => {
-              clearTimeout(timer);
-              reject(e);
-            });
-        });
-      }
-
-      if (!window.server_bridge_invoke) {
-        throw new Error('bridge 未注入 window.server_bridge_invoke');
-      }
-
-      return await with_timeout(window.server_bridge_invoke(action, payload), timeout);
-    },
-    action_name,
-    action_payload || {},
-    timeout_ms
-  );
-
-  await page.close();
-  return action_res;
-}

server/services/task_executor.js

@@ -1,6 +1,6 @@
 import { crawl_run_record } from '../models/index.js';
 import { safe_json_stringify } from './json_utils.js';
-import { invoke_extension_action } from './puppeteer_runner.js';
+import { invoke_extension_action } from './puppeteer/puppeteer_runner.js';
 import { persist_amazon_result } from './amazon_persist.js';
 
 export async function execute_action_and_record(params) {
@@ -13,18 +13,15 @@ export async function execute_action_and_record(params) {
   let error_message = null;
 
   try {
+    const result = await invoke_extension_action(action_name, action_payload || {});
 
-
-    console.log( 'invoke_extension_action-start', action_name, action_payload );
-    const res_invoke = await invoke_extension_action(action_name, action_payload || {});
-    console.log( 'invoke_extension_action-end', action_name, result );
     ok = true;
-    result_payload = safe_json_stringify(res_invoke);
+    result_payload = safe_json_stringify(result);
 
     // 按 stage 自动入库（不影响原始 run_record 记录）
-    await persist_amazon_result(res_invoke.result);
+    await persist_amazon_result(result);
 
-    return res_invoke;
+    return result;
   } catch (err) {
     ok = false;
     error_message = (err && err.message) || String(err);