Compare commits

...

18 Commits

Author SHA1 Message Date
7164dc3ed5 1 2026-03-26 20:56:06 +08:00
张成
6200fcc66f 1 2026-03-25 15:55:55 +08:00
张成
7ba462aedd 1 2026-03-20 10:55:55 +08:00
张成
873fd436b1 1 2026-03-19 18:44:18 +08:00
张成
66df49a7c3 1 2026-03-19 18:36:41 +08:00
张成
a776873477 1 2026-03-19 17:54:30 +08:00
张成
54cd6ea9ae 1 2026-03-19 16:35:57 +08:00
3d5156ac3e 1 2026-03-19 14:45:31 +08:00
cf3422b1ca 1 2026-03-18 20:43:42 +08:00
92021eaa41 11 2026-03-18 20:27:56 +08:00
张成
d75c9873c3 1 2026-03-18 18:12:07 +08:00
张成
a3db91b685 1 2026-03-18 18:11:04 +08:00
张成
aecb7944a8 1 2026-03-18 18:07:41 +08:00
张成
18aa083c91 1 2026-03-18 17:42:38 +08:00
张成
bc1068ec66 1 2026-03-18 16:57:22 +08:00
张成
7b42ee8ef5 1 2026-03-18 16:40:09 +08:00
张成
6d75720a89 1 2026-03-18 16:28:26 +08:00
张成
30d127ac0b 1 2026-03-18 16:16:17 +08:00
42 changed files with 4987 additions and 1356 deletions

5
.vscode/launch.json vendored
View File

@@ -24,7 +24,8 @@
],
// 工作区根目录
"cwd": "${workspaceFolder}\\server",
"program": "${workspaceFolder}\\server\\app.js"
"program": "${workspaceFolder}\\server\\app.js",
"args": ["--run_cron_now"]
}
]
}
}

View File

@@ -41,4 +41,7 @@
"C:\Program Files\Google\Chrome\Application\chrome.exe" --disable-features=ExtensionManifestV2Unsupported,ExtensionManifestV2Disabled
"C:\Program Files\Google\Chrome\Application\chrome.exe" --disable-features=ExtensionManifestV2Unsupported,ExtensionManifestV2Disabled
1.浏览器下载地址

View File

@@ -0,0 +1,163 @@
# 模块化重构说明
## 🎯 重构目标
使用现代 ES6 模块化方式,将分散的独立函数整合到统一的导出文件中,提供更友好的开发体验。
## 📁 新的项目结构
```
src/
├── libs/
│ ├── index.js # 统一导出所有库函数
│ ├── action_response.js
│ ├── tabs.js
│ └── action_meta.js
├── actions/
│ ├── index.js # 统一导出所有动作
│ ├── amazon.js
│ └── amazon_tool.js
├── background/
│ └── index.js # 使用新的导入方式
└── examples/
└── usage_example.js # 使用示例
```
## 🚀 使用方式
### 1. 命名导入(推荐)
```javascript
import {
ok_response,
fail_response,
create_tab_task,
getAllActionsMeta,
getActionByName
} from './libs/index.js';
```
### 2. 默认导入使用对象
```javascript
import Libs from './libs/index.js';
import Actions from './actions/index.js';
// 使用
const response = Libs.response.ok({ data: 'success' });
const task = Libs.tabs.createTask('https://example.com');
const actions = Actions.amazon;
```
### 3. 混合使用
```javascript
// 直接需要的函数用命名导入
import { ok_response, create_tab_task } from './libs/index.js';
// 复杂对象用默认导入
import Actions from './actions/index.js';
const action = getActionByName('amazon_search_list');
```
## 📦 统一导出内容
### libs/index.js
```javascript
// 响应处理
export {
ok_response,
fail_response,
response_code,
guard_sync
} from './action_response.js';
// Tab 操作
export {
raw_execute_script,
inject_file,
ensure_injected,
execute_script,
open_tab,
close_tab,
create_tab_task
} from './tabs.js';
// 元数据处理
export {
bind_action_meta
} from './action_meta.js';
// 便捷对象
export const Libs = {
response: { ok: ok_response, fail: fail_response, ... },
tabs: { open: open_tab, close: close_tab, ... },
meta: { bindAction: bind_action_meta }
};
```
### actions/index.js
```javascript
// 导出所有动作
export { amazon_actions } from './amazon.js';
// 导出工具函数
export {
injected_amazon_validate_captcha_continue,
run_amazon_pdp_action,
// ... 其他工具函数
} from './amazon_tool.js';
// 便捷函数
export function getAllActionsMeta() { ... }
export function getActionByName(name) { ... }
// 便捷对象
export const Actions = {
amazon: amazon_actions,
amazonTools: { ... }
};
```
## ✅ 优势
1. **统一入口** - 不需要记住具体的文件路径
2. **灵活导入** - 支持命名导入、默认导入、混合导入
3. **便于维护** - 集中管理所有导出
4. **向后兼容** - 保持原有功能不变
5. **现代语法** - 使用最新的 ES6 模块特性
## 🔄 迁移指南
### 旧方式
```javascript
import { create_tab_task } from '../libs/tabs.js';
import { ok_response } from '../libs/action_response.js';
import { amazon_actions } from '../actions/amazon.js';
```
### 新方式
```javascript
import { create_tab_task, ok_response, amazon_actions } from '../libs/index.js';
// 或者
import { create_tab_task, ok_response } from '../libs/index.js';
import { amazon_actions } from '../actions/index.js';
```
## 🎨 最佳实践
1. **简单函数** - 使用命名导入
2. **复杂对象** - 使用默认导入
3. **类型安全** - 配合 TypeScript 使用
4. **按需导入** - 只导入需要的功能
5. **统一风格** - 在一个项目中保持一致的导入风格
## 🔧 开发建议
- 新增功能时,优先添加到对应的 `index.js` 文件
- 保持导出名称的一致性和可读性
- 使用 JSDoc 注释提高代码可读性
- 定期检查和优化导出结构

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,429 @@
import { sleep_ms, get_tab_url } from '../libs/index.js';
// Amazon页面注入函数 + 纯工具
//
// 约定:
// - injected_* 在页面上下文执行,只依赖 DOM
// - 闭包外变量不会进入页面,辅助函数只能写在各 injected_* 函数体内
// ──────────── 验证码相关 ────────────
export function injected_amazon_validate_captcha_continue() {
const injected_utils = () => window.__mv2_simple_injected || null;
const dispatch_human_click = (target_el) => {
const u = injected_utils();
if (u && typeof u.dispatch_human_click === 'function') return u.dispatch_human_click(target_el);
return false;
};
// 基于 DOM 特征判断验证码页
const form = document.querySelector('form[action*="/errors/validateCaptcha"]');
if (!form) return { ok: true, is_captcha: false, clicked: false, href: location.href };
const btn =
form.querySelector('button[type="submit"].a-button-text') ||
form.querySelector('input[type="submit"]') ||
form.querySelector('button[type="submit"]') ||
document.querySelector('input[type="submit"][value*="Continue"]') ||
document.querySelector('button[type="submit"]');
const clicked = btn ? dispatch_human_click(btn) : false;
if (!clicked) {
try {
form.submit();
return { ok: true, is_captcha: true, clicked: true, method: 'submit', href: location.href };
} catch (_) {
return { ok: false, is_captcha: true, clicked: false, method: 'submit', href: location.href };
}
}
return { ok: true, is_captcha: true, clicked, method: clicked ? 'dispatch' : 'none', href: location.href };
}
export function is_amazon_validate_captcha_url(tab_url) {
if (!tab_url || typeof tab_url !== 'string') return false;
return tab_url.includes('amazon.') && tab_url.includes('/errors/validateCaptcha');
}
/**
* 基于 DOM 特征检测验证码页(注入到页面执行)
*/
export function injected_detect_captcha_page() {
const form = document.querySelector('form[action*="/errors/validateCaptcha"]');
const btn = document.querySelector(
'form[action*="/errors/validateCaptcha"] button[type="submit"], form[action*="/errors/validateCaptcha"] input[type="submit"]'
);
const has_continue_h4 = Array.from(document.querySelectorAll('h4')).some((el) => {
const txt = (el.textContent || '').trim().toLowerCase();
return txt.includes('continue shopping');
});
return !!(form && (btn || has_continue_h4));
}
export async function try_solve_amazon_validate_captcha(tab, max_round) {
const rounds = Number.isFinite(max_round) ? Math.max(1, Math.min(5, Math.floor(max_round))) : 2;
for (let i = 0; i < rounds; i += 1) {
const url = await get_tab_url(tab.id).catch(() => '');
if (!is_amazon_validate_captcha_url(url)) return true;
await tab.execute_script(injected_amazon_validate_captcha_continue, [], 'document_idle');
await sleep_ms(800 + Math.floor(Math.random() * 600));
await tab.wait_complete();
await sleep_ms(300);
}
return false;
}
// ──────────── 结果工具 ────────────
export function pick_first_script_result(raw_list) {
if (!Array.isArray(raw_list) || raw_list.length === 0) return null;
const first = raw_list[0];
if (first && typeof first === 'object' && Object.prototype.hasOwnProperty.call(first, 'result')) {
return first.result;
}
return first;
}
// ──────────── 首页搜索(注入) ────────────
export function injected_amazon_homepage_search(params) {
const injected_utils = () => window.__mv2_simple_injected || null;
const dispatch_human_click = (target_el) => {
const u = injected_utils();
if (u && typeof u.dispatch_human_click === 'function') return u.dispatch_human_click(target_el);
return false;
};
const keyword = params && params.keyword ? String(params.keyword).trim() : '';
if (!keyword) return { ok: false, error: 'empty_keyword' };
const u = injected_utils();
const wait_query = u && typeof u.wait_query === 'function' ? u.wait_query : () => null;
const set_input_value = u && typeof u.set_input_value === 'function' ? u.set_input_value : () => false;
const input = wait_query([
'#twotabsearchtextbox',
'input#nav-search-keywords',
'input[name="field-keywords"]',
'input[type="search"][name="field-keywords"]',
], 7000);
if (!input) return { ok: false, error: 'no_search_input' };
set_input_value(input, keyword);
const btn = wait_query([
'#nav-search-submit-button',
'#nav-search-bar-form input[type="submit"]',
'#nav-search-bar-form button[type="submit"]',
'form[role="search"] input[type="submit"]',
'form[role="search"] button[type="submit"]',
'input.nav-input[type="submit"]',
], 2000);
if (btn) {
const clicked = dispatch_human_click(btn);
if (clicked) return { ok: true, method: 'button_click' };
}
const form = input.form || input.closest('form');
if (form && typeof form.requestSubmit === 'function') {
try { form.requestSubmit(); return { ok: true, method: 'request_submit' }; } catch (_) {}
}
if (form && typeof form.submit === 'function') {
try { form.submit(); return { ok: true, method: 'form_submit' }; } catch (_) {}
}
try {
input.focus();
input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true }));
input.dispatchEvent(new KeyboardEvent('keypress', { key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true }));
input.dispatchEvent(new KeyboardEvent('keyup', { key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true }));
if (form) form.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
return { ok: true, method: 'keyboard_enter' };
} catch (_) {}
return { ok: false, error: 'submit_all_fallback_failed', keyword };
}
// ──────────── 切换语言(注入) ────────────
export function injected_amazon_switch_language(params) {
const injected_utils = () => window.__mv2_simple_injected || null;
const dispatch_human_click = (target_el) => {
const u = injected_utils();
if (u && typeof u.dispatch_human_click === 'function') return u.dispatch_human_click(target_el);
return false;
};
const mapping = {
EN: 'en_US', ES: 'es_US', AR: 'ar_AE', DE: 'de_DE',
HE: 'he_IL', KO: 'ko_KR', PT: 'pt_BR', ZH_CN: 'zh_CN', ZH_TW: 'zh_TW',
};
const raw = params && params.lang != null ? String(params.lang).trim().toUpperCase() : 'ZH_CN';
const code = Object.prototype.hasOwnProperty.call(mapping, raw) ? raw : 'ZH_CN';
const switch_lang = mapping[code];
const href_sel = `a[href="#switch-lang=${switch_lang}"]`;
const u = injected_utils();
const deadline = Date.now() + 6000;
let link = null;
while (Date.now() < deadline) {
link = document.querySelector(href_sel);
if (link) {
const r = link.getBoundingClientRect();
if (r.width > 0 && r.height > 0) break;
}
if (u && typeof u.busy_wait_ms === 'function') u.busy_wait_ms(40);
}
if (!link) return { ok: false, error: 'lang_option_timeout', lang: code };
dispatch_human_click(link);
const save_deadline = Date.now() + 6000;
let save = null;
while (Date.now() < save_deadline) {
save =
document.querySelector('input[type="submit"][value*="Save"]') ||
document.querySelector('input[type="submit"][aria-labelledby*="icp-save-button"]') ||
document.querySelector('span.icp-save-button input[type="submit"]');
if (save) break;
if (u && typeof u.busy_wait_ms === 'function') u.busy_wait_ms(40);
}
if (save) dispatch_human_click(save);
return { ok: true, lang: code };
}
// ──────────── 搜索列表(注入) ────────────
export function injected_amazon_search_list(params) {
const injected_utils = () => window.__mv2_simple_injected || null;
const dispatch_human_click = (target_el) => {
const u = injected_utils();
if (u && typeof u.dispatch_human_click === 'function') return u.dispatch_human_click(target_el);
return false;
};
params = params && typeof params === 'object' ? params : {};
const debug = params.debug === true;
const u = injected_utils();
// validateCaptcha 页面:直接点击继续
if ((location.href || '').includes('/errors/validateCaptcha')) {
const btn =
document.querySelector('form[action="/errors/validateCaptcha"] button[type="submit"].a-button-text') ||
document.querySelector('form[action*="validateCaptcha"] input[type="submit"]') ||
document.querySelector('form[action*="validateCaptcha"] button[type="submit"]') ||
document.querySelector('input[type="submit"][value*="Continue"]') ||
document.querySelector('button[type="submit"]');
const clicked = btn ? dispatch_human_click(btn) : false;
if (debug) console.log('[amazon][on_complete] validateCaptcha', { clicked, href: location.href });
return { stage: 'captcha', href: location.href, clicked };
}
const start_url = params.url ? String(params.url) : location.href;
const category_keyword = params.category_keyword ? String(params.category_keyword).trim() : '';
const sort_by = params.sort_by ? String(params.sort_by).trim() : '';
const abs_url = u && typeof u.abs_url === 'function' ? u.abs_url : (x) => x;
const parse_asin_from_url = u && typeof u.parse_asin_from_url === 'function' ? u.parse_asin_from_url : () => null;
const pick_number = u && typeof u.pick_number === 'function' ? u.pick_number : () => null;
const pick_int = u && typeof u.pick_int === 'function' ? u.pick_int : () => null;
function extract_results() {
const items = [];
const nodes = document.querySelectorAll('div.s-main-slot div[data-component-type="s-search-result"]');
nodes.forEach((el, idx) => {
const asin = (el.getAttribute('data-asin') || '').trim() || null;
const title_el = el.querySelector('h2 span') || el.querySelector('h2');
const title = title_el ? title_el.textContent.trim() : null;
const a = el.querySelector('a[href*="/dp/"], a[href*="/gp/product/"]');
const href = a ? a.getAttribute('href') : null;
const item_url = href ? abs_url(href) : null;
const price_el = el.querySelector('span.a-price > span.a-offscreen');
const price = price_el ? price_el.textContent.trim() : null;
const reviews_block = el.querySelector('div[data-cy="reviews-block"]') || el;
const rating_text = (() => {
const t1 = reviews_block.querySelector('span.a-icon-alt');
if (t1 && t1.textContent) return t1.textContent.trim();
const t2 = reviews_block.querySelector('span.a-size-small.a-color-base[aria-hidden="true"]');
if (t2 && t2.textContent) return t2.textContent.trim();
return null;
})();
const rating = (() => {
const n = pick_number(rating_text);
return Number.isFinite(n) ? n : null;
})();
const review_count_text = (() => {
const a1 = reviews_block.querySelector('a[href*="#customerReviews"]');
if (a1 && a1.textContent) return a1.textContent.trim();
const a2 = reviews_block.querySelector(
'a[aria-label*="rating"], a[aria-label*="ratings"], a[aria-label*="评级"], a[aria-label*="评价"]',
);
if (a2 && a2.getAttribute('aria-label')) return a2.getAttribute('aria-label').trim();
const s1 = reviews_block.querySelector('span.a-size-mini.puis-normal-weight-text');
if (s1 && s1.textContent) return s1.textContent.trim();
return null;
})();
const review_count = (() => {
const n = pick_int(review_count_text);
return Number.isFinite(n) ? n : null;
})();
items.push({
index: idx + 1,
asin: asin || parse_asin_from_url(item_url),
title, url: item_url, price,
rating, rating_text, review_count, review_count_text,
});
});
return items;
}
function pick_next_url() {
const a = document.querySelector('a.s-pagination-next');
if (!a) return null;
if ((a.getAttribute('aria-disabled') || '').trim().toLowerCase() === 'true') return null;
if (a.classList && a.classList.contains('s-pagination-disabled')) return null;
const href = a.getAttribute('href');
return href ? abs_url(href) : null;
}
const items = extract_results();
const out = { start_url, href: location.href, category_keyword, sort_by, total: items.length, items, next_url: pick_next_url() };
if (debug) {
console.log('[amazon][on_complete] search_list', { href: out.href, total: out.total, has_next: !!out.next_url });
try { window.__amazon_debug_last_search_list = out; } catch (_) {}
}
return out;
}
// ──────────── 商品详情(注入) ────────────
export function injected_amazon_product_detail() {
const u = window.__mv2_simple_injected || null;
const norm = u && typeof u.norm_space === 'function' ? u.norm_space : (s) => (s || '').replace(/\s+/g, ' ').trim();
const asin_match = location.pathname.match(/\/(?:dp|gp\/product)\/([A-Z0-9]{10})/i);
const asin = asin_match ? asin_match[1].toUpperCase() : null;
const product_info = {};
function set_info(k, v, max_len) {
k = norm(k); v = norm(v); max_len = max_len || 600;
if (!k || !v || k.length > 100) return;
if (v.length > max_len) v = v.slice(0, max_len);
if (!product_info[k] || v.length > product_info[k].length) product_info[k] = v;
}
const table_roots =
'#productOverview_feature_div tr, #poExpander table tr, #productDetails_detailBullets_sections1 tr, ' +
'#productDetails_techSpec_section_1 tr, table.prodDetTable tr, #productFactsDesktopExpander tr, ' +
'#technicalSpecifications_feature_div table tr, #productDetails_db_sections tr';
document.querySelectorAll(table_roots).forEach((tr) => {
const tds = tr.querySelectorAll('td');
const th = tr.querySelector('th');
const td = tr.querySelector('td');
if (tds.length >= 2) set_info(tds[0].innerText, tds[1].innerText);
else if (th && td && th !== td) set_info(th.innerText, td.innerText);
});
const detail_extra_lines = [];
document.querySelectorAll('#detailBullets_feature_div li, #rpi-attribute-values_feature_div li').forEach((li) => {
const t = li.innerText.replace(/\u200f|\u200e/g, ' ').replace(/\s+/g, ' ').trim();
const m = t.match(/^(.{1,80}?)\s*[:]\s*(.+)$/);
if (m) set_info(m[1], m[2], 1200);
else if (t.length > 8 && t.length < 800) detail_extra_lines.push(t);
});
const title_el = document.querySelector('#productTitle');
const title = title_el ? norm(title_el.textContent) : null;
const price_el =
document.querySelector('#corePrice_feature_div .a-price .a-offscreen') ||
document.querySelector('#tp_price_block_total_price_ww .a-offscreen') ||
document.querySelector('#price .a-offscreen') ||
document.querySelector('.reinventPricePriceToPayMargin .a-offscreen') ||
document.querySelector('.a-price .a-offscreen');
const price = price_el ? price_el.textContent.trim() : null;
const brand_el = document.querySelector('#bylineInfo');
const brand_line = brand_el ? norm(brand_el.textContent) : null;
const brand_store_url = document.querySelector('#bylineInfo a[href]')?.href || null;
const rating_stars = document.querySelector('#acrPopover')?.getAttribute('title') ||
document.querySelector('#averageCustomerReviews .a-icon-alt')?.textContent?.trim() || null;
const review_count_text = document.querySelector('#acrCustomerReviewText')?.textContent?.trim() || null;
const ac_badge = norm(document.querySelector('#acBadge_feature_div')?.innerText) || null;
const social_proof = norm(document.querySelector('#socialProofingAsinFaceout_feature_div')?.innerText) || null;
const bestseller_hint = norm(document.querySelector('#zeitgeistBadge_feature_div')?.innerText)?.slice(0, 200) || null;
const bullets = [];
document.querySelectorAll('#feature-bullets ul li span.a-list-item').forEach((el) => {
const t = norm(el.textContent);
if (t) bullets.push(t);
});
let delivery_hint = null;
const del = document.querySelector('#deliveryBlockMessage, #mir-layout-DELIVERY_BLOCK-slot-PRIMARY_DELIVERY_MESSAGE_LARGE');
if (del) delivery_hint = norm(del.innerText).slice(0, 500);
return {
stage: 'detail', asin, title, price,
brand_line, brand_store_url, rating_stars, review_count_text,
ac_badge, social_proof, bestseller_hint,
product_info, detail_extra_lines, bullets, delivery_hint,
url: location.href,
};
}
// ──────────── 商品评论(注入) ────────────
export function injected_amazon_product_reviews(params) {
const raw = params && params.limit != null ? Number(params.limit) : 50;
const limit = Number.isFinite(raw) ? Math.max(1, Math.min(100, Math.floor(raw))) : 50;
const nodes = document.querySelectorAll('[data-hook="review"]');
const items = [];
nodes.forEach((r) => {
if (items.length >= limit) return;
const author_el = r.querySelector('.a-profile-name');
const author = author_el ? author_el.textContent.trim() : null;
const title_el = r.querySelector('[data-hook="review-title"]');
const title = title_el ? title_el.innerText.replace(/\s+/g, ' ').trim() : null;
const body_el = r.querySelector('[data-hook="review-body"]');
const body = body_el ? body_el.innerText.replace(/\s+/g, ' ').trim() : null;
const rating_el = r.querySelector('[data-hook="review-star-rating"]');
const rating_text = rating_el ? rating_el.textContent.trim() : null;
const date_el = r.querySelector('[data-hook="review-date"]');
const date = date_el ? date_el.textContent.trim() : null;
const cr = r.querySelector('[id^="customer_review-"]');
const review_id = r.id || (cr && cr.id ? cr.id.replace('customer_review-', '') : null);
items.push({ index: items.length + 1, review_id, author, rating_text, title, date, body });
});
return { stage: 'reviews', limit, total: items.length, items, url: location.href };
}
// ──────────── URL 工具 ────────────
export function normalize_product_url(u) {
let s = u ? String(u).trim() : '';
if (!s) throw new Error('缺少 product_url');
if (s.startsWith('//')) s = 'https:' + s;
if (!/^https?:\/\//i.test(s)) s = 'https://' + s;
const url_obj = new URL(s);
if (!url_obj.hostname.includes('amazon.')) throw new Error('product_url 需为亚马逊域名');
if (!/\/dp\/[A-Z0-9]{10}/i.test(url_obj.pathname) && !/\/gp\/product\/[A-Z0-9]{10}/i.test(url_obj.pathname)) {
throw new Error('product_url 需包含 /dp/ASIN 或 /gp/product/ASIN');
}
return url_obj.toString();
}
export function is_amazon_search_list_url(tab_url) {
if (!tab_url || typeof tab_url !== 'string') return false;
if (!tab_url.includes('amazon.com')) return false;
if (!/\/s(\?|\/)/.test(tab_url)) return false;
return tab_url.includes('k=') || tab_url.includes('keywords=') || tab_url.includes('field-keywords');
}
/**
* 轮询等待 tab URL 变为搜索列表页async 循环,替代旧版回调递归)
*/
export async function wait_until_search_list_url(tab_id, timeout_ms = 45000) {
const deadline = Date.now() + timeout_ms;
while (Date.now() < deadline) {
const url = await get_tab_url(tab_id);
if (is_amazon_search_list_url(url)) return url;
await sleep_ms(400);
}
throw new Error('等待首页搜索跳转到列表页超时');
}

View File

@@ -0,0 +1,38 @@
/**
* 统一的动作导出
* 使用现代 ES6 模块化方式,提供统一的动作接口
*/
// Amazon 相关动作
import { amazon_actions } from './amazon.js';
export { amazon_actions };
// 获取所有动作的元信息
export function getAllActionsMeta() {
const meta = {};
if (Array.isArray(amazon_actions)) {
amazon_actions.forEach((item) => {
if (item && item.name) {
meta[item.name] = {
name: item.name,
desc: item.desc || '',
params: item.params || {},
};
}
});
}
return meta;
}
// 根据名称获取动作
export function getActionByName(name) {
if (!Array.isArray(amazon_actions)) return null;
return amazon_actions.find(item => item && item.name === name);
}
export default {
amazon: amazon_actions,
};;

View File

@@ -1,32 +1,43 @@
import { amazon_actions, getAllActionsMeta, getActionByName } from '../actions/index.js';
import {
amazon_search_list,
amazon_set_language,
amazon_product_detail,
amazon_product_reviews,
} from '../actions/amazon.js';
// ──────────── Action 注册 ────────────
const actions = {
amazon_search_list,
amazon_set_language,
amazon_product_detail,
amazon_product_reviews,
};
function list_actions_meta() {
const meta = {};
Object.keys(actions).forEach((name) => {
const fn = actions[name];
meta[name] = {
name,
desc: fn && fn.desc ? fn.desc : '',
params: fn && fn.params ? fn.params : {},
};
});
return meta;
let action_list = [];
try {
if (Array.isArray(amazon_actions)) {
action_list = amazon_actions.filter(item => item && typeof item === 'object' && item.name);
console.log(`Loaded ${action_list.length} actions:`, action_list.map(item => item.name));
} else {
console.warn('amazon_actions is not an array:', amazon_actions);
}
} catch (error) {
console.error('Failed to load amazon_actions:', error);
}
function create_action_send_response(sender) {
// ──────────── UI 事件推送 ────────────
const ui_page_url = chrome.runtime.getURL('ui/index.html');
const is_port_closed_error = (message) => {
const text = message ? String(message) : '';
return text.includes('The message port closed before a response was received');
};
const emit_ui_event = (event_name, payload) => {
try {
chrome.runtime.sendMessage({ channel: 'ui_event', event_name, payload, ts: Date.now() }, (response) => {
if (chrome.runtime.lastError) {
const err_msg = chrome.runtime.lastError.message;
if (is_port_closed_error(err_msg)) return;
console.warn('Failed to send UI event:', err_msg);
}
});
} catch (error) {
console.error('Error in emit_ui_event:', error);
}
};
const create_action_send_response = (sender) => {
const fn = (payload) => {
emit_ui_event('push', { type: 'reply', ...payload, sender });
};
@@ -34,92 +45,83 @@ function create_action_send_response(sender) {
emit_ui_event('push', { type: 'log', action: 'log', data: payload, sender });
};
return fn;
};
// ──────────── 内置 action 路由 ────────────
const builtin_handlers = {
meta_actions(message, sender, sendResponse) {
console.log('Returning actions meta');
sendResponse({ ok: true, data: getAllActionsMeta() });
},
reload_background(message, sender, sendResponse) {
console.log('Reloading background page');
sendResponse({ ok: true });
setTimeout(() => location.reload(), 50);
},
};
// ──────────── Action 执行器 ────────────
async function execute_action(action_handler, message, sender, sendResponse) {
const request_id = `${Date.now()}_${Math.random().toString().slice(2)}`;
console.log('Executing action:', { action: message.action, request_id, data: message.data });
emit_ui_event('request', { type: 'request', request_id, action: message.action, data: message.data || {}, sender });
const action_send_response = create_action_send_response(sender);
try {
const res = await action_handler(message.data || {}, action_send_response);
console.log(`Action ${message.action} completed successfully:`, { request_id, result: res });
emit_ui_event('response', { type: 'response', request_id, ok: true, data: res, sender });
sendResponse({ ok: true, data: res, request_id });
} catch (err) {
const error = (err && err.message) || String(err);
const stack = (err && err.stack) || '';
console.error(`Action ${message.action} failed:`, { error, stack, data: message.data });
emit_ui_event('response', { type: 'response', request_id, ok: false, error, stack, sender });
sendResponse({ ok: false, error, stack, request_id });
}
}
const ui_page_url = chrome.runtime.getURL('ui/index.html');
function log() {
// eslint-disable-next-line no-console
console.log.apply(console, ['[mv2_simple_crx]'].concat([].slice.call(arguments)));
}
function emit_ui_event(event_name, payload) {
chrome.runtime.sendMessage({
channel: 'ui_event',
event_name,
payload,
ts: Date.now(),
});
}
// ──────────── 消息分发 ────────────
chrome.browserAction.onClicked.addListener(() => {
chrome.tabs.create({ url: ui_page_url, active: true });
});
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
if (!message) {
return;
}
// 忽略空消息、UI 自身事件、page world 桥接
if (!message || message.channel === 'ui_event' || message.channel === 'page_exec_bridge') return;
// UI 自己发出来的事件,background 不处理
if (message.channel === 'ui_event') {
return;
}
// content -> background 的推送消息(通用)
// content -> background 推送
if (message.type === 'push') {
emit_ui_event('push', {
type: 'push',
action: message.action,
data: message.data,
sender,
});
console.log('Processing push message:', message.action);
emit_ui_event('push', { type: 'push', action: message.action, data: message.data, sender });
return;
}
// UI -> background 的 action 调用
// 缺少 action
if (!message.action) {
console.error('Missing action in message');
sendResponse && sendResponse({ ok: false, error: '缺少 action' });
return;
}
// UI 获取 action 元信息(用于下拉/默认参数
if (message.action === 'meta_actions') {
sendResponse({ ok: true, data: list_actions_meta() });
// 内置 action(同步处理,不需要 return true
if (builtin_handlers[message.action]) {
builtin_handlers[message.action](message, sender, sendResponse);
return;
}
// UI 刷新后台(重启 background page
if (message.action === 'reload_background') {
sendResponse({ ok: true });
setTimeout(() => {
location.reload();
}, 50);
return;
}
const fn = actions[message.action];
if (!fn) {
// 业务 action
const action_item = getActionByName(message.action);
if (!action_item || typeof action_item.handler !== 'function') {
console.error('Unknown action:', message.action);
sendResponse({ ok: false, error: '未知 action: ' + message.action });
return;
}
const request_id = `${Date.now()}_${Math.random().toString().slice(2)}`;
emit_ui_event('request', { type: 'request', request_id, action: message.action, data: message.data || {}, sender });
const action_send_response = create_action_send_response(sender);
(async () => {
try {
const res = await fn(message.data || {}, action_send_response);
emit_ui_event('response', { type: 'response', request_id, ok: true, data: res, sender });
sendResponse({ ok: true, data: res, request_id });
} catch (err) {
const error = (err && err.message) || String(err);
emit_ui_event('response', { type: 'response', request_id, ok: false, error, sender });
sendResponse({ ok: false, error, request_id });
}
})();
execute_action(action_item.handler, message, sender, sendResponse);
return true;
});

View File

@@ -2,31 +2,37 @@
* 服务端 Puppeteer 通过此页与 background 通讯(等同 UI 发 chrome.runtime.sendMessage
* 页面内若需 Web Worker 做重计算,可在此 postMessage当前直连 background 即可满足指令/结果
*/
(function () {
function server_bridge_invoke(action, data) {
return new Promise(function (resolve, reject) {
(() => {
const server_bridge_invoke = (action, data) => {
return new Promise((resolve, reject) => {
if (!action) {
reject(new Error('缺少 action'));
return;
}
chrome.runtime.sendMessage({ action: action, data: data || {} }, function (res) {
var err = chrome.runtime.lastError;
if (err) {
reject(new Error(err.message));
return;
chrome.runtime.sendMessage(
{ action, data: data || {} },
(res) => {
const err = chrome.runtime.lastError;
if (err) {
reject(new Error(err.message));
return;
}
if (!res) {
reject(new Error('background 无响应'));
return;
}
if (res.ok) {
resolve(res.data);
} else {
reject(new Error(res.error || 'action 失败'));
}
}
if (!res) {
reject(new Error('background 无响应'));
return;
}
if (res.ok) {
resolve(res.data);
} else {
reject(new Error(res.error || 'action 失败'));
}
});
);
});
}
};
window.server_bridge_invoke = server_bridge_invoke;
})();

View File

@@ -2,6 +2,29 @@
// 目标:页面里触发 XHR/fetch 时派发 __REQUEST_DONE
(() => {
function inject_page_file_once(file_path, marker) {
const root = document.documentElement || document.head;
if (!root) return false;
const mark_key = marker || file_path;
const attr = `data-mv2-injected-${mark_key.replace(/[^a-z0-9_-]/gi, '_')}`;
if (root.hasAttribute(attr)) return true;
const src = chrome.runtime.getURL(file_path);
const el = document.createElement('script');
el.type = 'text/javascript';
el.src = src;
el.onload = () => {
el.parentNode && el.parentNode.removeChild(el);
};
el.onerror = () => {
el.parentNode && el.parentNode.removeChild(el);
root.removeAttribute(attr);
};
root.setAttribute(attr, '1');
(document.head || document.documentElement).appendChild(el);
return true;
}
function inject_inline(fn) {
const el = document.createElement('script');
el.type = 'text/javascript';
@@ -161,5 +184,8 @@
F.__RequestWatcher = true;
}
// 页面上下文通用方法window.__mv2_simple_injected
inject_page_file_once('injected/injected.js', 'core_utils');
inject_inline(request_watcher);
})();

View File

@@ -0,0 +1,53 @@
/**
* 使用新模块化方式的示例
* 展示如何使用统一的导出接口
*/
// 方式1: 命名导入(推荐)
import { ok_response, fail_response, create_tab_task, amazon_actions, getAllActionsMeta, getActionByName } from '../libs/index.js';
// 方式2: 默认导入使用对象
import Libs from '../libs/index.js';
import Actions from '../actions/index.js';
// 示例函数
export async function exampleAction() {
// 使用命名导入
const response = ok_response({ success: true });
// 使用默认导入
const task = Libs.tabs.createTask('https://example.com');
// 使用 Actions
const allMeta = getAllActionsMeta();
const specificAction = getActionByName('amazon_search_list');
return {
response,
task,
allMeta,
specificAction
};
}
// 更简洁的写法
export const ModernUsage = {
// 响应处理
response: {
success: (data) => ok_response(data),
error: (msg) => fail_response(msg)
},
// Tab 操作
tabs: {
create: (url) => create_tab_task(url),
// ... 其他操作
},
// 动作管理
actions: {
getAll: getAllActionsMeta,
get: getActionByName,
list: amazon_actions
}
};

View File

@@ -0,0 +1,148 @@
(() => {
if (window.__mv2_simple_injected) return;
const norm_space = (s) => (s || '').toString().replace(/\s+/g, ' ').trim();
const busy_wait_ms = (ms) => {
const t = Number(ms);
const dur = Number.isFinite(t) ? Math.max(0, t) : 0;
const t0 = performance.now();
while (performance.now() - t0 < dur) { }
};
const is_visible = (el) => {
if (!el) return false;
const r = el.getBoundingClientRect();
if (!(r.width > 0 && r.height > 0)) return false;
// 尽量避免点击到不可见层display/visibility 由浏览器计算
const cs = window.getComputedStyle(el);
if (!cs) return true;
if (cs.display === 'none') return false;
if (cs.visibility === 'hidden') return false;
if (cs.opacity === '0') return false;
return true;
};
const wait_query = (selectors, timeout_ms) => {
const list = Array.isArray(selectors) ? selectors : [];
const deadline = Date.now() + (Number.isFinite(timeout_ms) ? timeout_ms : 5000);
while (Date.now() < deadline) {
for (let i = 0; i < list.length; i += 1) {
const sel = list[i];
const el = document.querySelector(sel);
if (is_visible(el)) return el;
}
busy_wait_ms(40);
}
return null;
};
const dispatch_human_click = (target_el, options) => {
const el = target_el;
if (!el) return false;
const opt = options && typeof options === 'object' ? options : {};
const pointer_id = Number.isFinite(opt.pointer_id) ? opt.pointer_id : 1;
const pointer_type = opt.pointer_type ? String(opt.pointer_type) : 'mouse';
try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch (_) { }
try { el.focus && el.focus(); } catch (_) { }
const rect = el.getBoundingClientRect();
const ox = Number.isFinite(opt.offset_x) ? opt.offset_x : 0;
const oy = Number.isFinite(opt.offset_y) ? opt.offset_y : 0;
const x = Math.max(1, Math.floor(rect.left + rect.width / 2 + ox));
const y = Math.max(1, Math.floor(rect.top + rect.height / 2 + oy));
const base = { bubbles: true, cancelable: true, view: window, clientX: x, clientY: y };
try {
if (typeof PointerEvent === 'function') {
el.dispatchEvent(new PointerEvent('pointerover', { ...base, pointerId: pointer_id, pointerType: pointer_type, isPrimary: true }));
el.dispatchEvent(new PointerEvent('pointerenter', { ...base, pointerId: pointer_id, pointerType: pointer_type, isPrimary: true }));
el.dispatchEvent(new PointerEvent('pointermove', { ...base, pointerId: pointer_id, pointerType: pointer_type, isPrimary: true }));
el.dispatchEvent(new PointerEvent('pointerdown', { ...base, pointerId: pointer_id, pointerType: pointer_type, isPrimary: true, buttons: 1 }));
el.dispatchEvent(new PointerEvent('pointerup', { ...base, pointerId: pointer_id, pointerType: pointer_type, isPrimary: true, buttons: 0 }));
}
} catch (_) { }
el.dispatchEvent(new MouseEvent('mousemove', base));
el.dispatchEvent(new MouseEvent('mouseover', base));
el.dispatchEvent(new MouseEvent('mousedown', base));
el.dispatchEvent(new MouseEvent('mouseup', base));
el.dispatchEvent(new MouseEvent('click', base));
return true;
};
const text = (el) => el && el.textContent != null ? norm_space(el.textContent) : null;
const inner_text = (el) => el && el.innerText != null ? norm_space(el.innerText) : null;
const attr = (el, name) => {
if (!el || !name) return null;
const v = el.getAttribute ? el.getAttribute(name) : null;
return v != null ? norm_space(v) : null;
};
const abs_url = (href, base) => {
try {
return new URL(href, base || location.origin).toString();
} catch (_) {
return href;
}
};
const parse_asin_from_url = (url) => {
if (!url || typeof url !== 'string') return null;
const m = url.match(/\/dp\/([A-Z0-9]{10})/i) || url.match(/\/gp\/product\/([A-Z0-9]{10})/i);
return m ? m[1].toUpperCase() : null;
};
const pick_number = (text0) => {
if (!text0) return null;
const s = String(text0).replace(/[(),]/g, ' ').replace(/\s+/g, ' ').trim();
const m = s.match(/(\d+(?:\.\d+)?)/);
return m ? Number(m[1]) : null;
};
const pick_int = (text0) => {
if (!text0) return null;
const raw = String(text0).replace(/\s+/g, ' ').trim();
const up = raw.toUpperCase().replace(/,/g, '');
const km = up.match(/([\d.]+)\s*K\b/);
if (km) return Math.round(parseFloat(km[1]) * 1000);
const mm = up.match(/([\d.]+)\s*M\b/);
if (mm) return Math.round(parseFloat(mm[1]) * 1000000);
const digits = raw.replace(/[^\d]/g, '');
return digits ? Number(digits) : null;
};
const set_input_value = (input, value, options) => {
if (!input) return false;
const opt = options && typeof options === 'object' ? options : {};
try { input.focus && input.focus(); } catch (_) { }
try { input.value = value == null ? '' : String(value); } catch (_) { return false; }
if (opt.dispatch_input !== false) {
try { input.dispatchEvent(new Event('input', { bubbles: true })); } catch (_) { }
}
if (opt.dispatch_change !== false) {
try { input.dispatchEvent(new Event('change', { bubbles: true })); } catch (_) { }
}
return true;
};
window.__mv2_simple_injected = {
norm_space: norm_space,
busy_wait_ms: busy_wait_ms,
wait_query: wait_query,
is_visible: is_visible,
dispatch_human_click: dispatch_human_click,
text: text,
inner_text: inner_text,
attr: attr,
abs_url: abs_url,
parse_asin_from_url: parse_asin_from_url,
pick_number: pick_number,
pick_int: pick_int,
set_input_value: set_input_value,
};
})();

View File

@@ -0,0 +1,13 @@
// 统一绑定 action 元数据:集中配置,同时兼容历史 fn.desc/fn.params 读取方式。
export function bind_action_meta(action_map, meta_map) {
const actions = action_map && typeof action_map === 'object' ? action_map : {};
const metas = meta_map && typeof meta_map === 'object' ? meta_map : {};
Object.keys(metas).forEach((action_name) => {
const action_fn = actions[action_name];
const meta = metas[action_name] || {};
if (typeof action_fn !== 'function') return;
action_fn.desc = meta.desc || '';
action_fn.params = meta.params || {};
});
return metas;
}

View File

@@ -0,0 +1,44 @@
const RESPONSE_CODE_OK = 0;
const RESPONSE_CODE_BAD_REQUEST = 10;
const RESPONSE_CODE_RUNTIME_ERROR = 30;
// 成功响应工厂:统一返回结构与成功码。
export function ok_response(data) {
return {
code: RESPONSE_CODE_OK,
status: true,
message: 'ok',
data: data == null ? null : data,
};
}
// 失败响应工厂:统一错误码、错误消息和可选上下文。
export function fail_response(message, options) {
const opts = options && typeof options === 'object' ? options : {};
const code = Number.isFinite(opts.code) ? Number(opts.code) : RESPONSE_CODE_RUNTIME_ERROR;
const data = Object.prototype.hasOwnProperty.call(opts, 'data') ? opts.data : null;
const documentURI = Object.prototype.hasOwnProperty.call(opts, 'documentURI') ? opts.documentURI : undefined;
return {
code,
status: false,
message: message ? String(message) : 'error',
data,
...(documentURI ? { documentURI } : {}),
};
}
// 响应码常量:供业务层统一引用,避免魔法数字。
export const response_code = {
ok: RESPONSE_CODE_OK,
bad_request: RESPONSE_CODE_BAD_REQUEST,
runtime_error: RESPONSE_CODE_RUNTIME_ERROR,
};
// 同步执行保护:把同步异常转为统一结果对象,避免业务层到处写 try/catch。
export function guard_sync(task) {
try {
return { ok: true, data: task() };
} catch (error) {
return { ok: false, error };
}
}

View File

@@ -0,0 +1,61 @@
/**
* 统一的库函数导出
* 使用现代 ES6 模块化方式,提供统一的功能接口
*/
// 响应处理相关
import { ok_response, fail_response, response_code, guard_sync } from './action_response.js';
export { ok_response, fail_response, response_code, guard_sync };
// Tab 操作相关
import { raw_execute_script, inject_file, ensure_injected, execute_script, open_tab, close_tab, create_tab_task } from './tabs.js';
export { raw_execute_script, inject_file, ensure_injected, execute_script, open_tab, close_tab, create_tab_task };
// Action 元数据相关
import { bind_action_meta } from './action_meta.js';
export { bind_action_meta };
// 通用异步工具
export function sleep_ms(ms) {
const t = Number(ms);
return new Promise((resolve) => setTimeout(resolve, Number.isFinite(t) ? Math.max(0, t) : 0));
}
export async function get_tab_url(tab_id) {
return await new Promise((resolve, reject) => {
chrome.tabs.get(tab_id, (tab) => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(tab && tab.url ? String(tab.url) : '');
});
});
}
// 便捷的统一导出对象(可选使用)
export const Libs = {
// 响应处理
response: {
ok: ok_response,
fail: fail_response,
code: response_code,
guard: guard_sync
},
// Tab 操作
tabs: {
rawExecuteScript: raw_execute_script,
injectFile: inject_file,
ensureInjected: ensure_injected,
executeScript: execute_script,
open: open_tab,
close: close_tab,
createTask: create_tab_task
},
// 元数据
meta: {
bindAction: bind_action_meta
}
};
// 默认导出(可选使用)
export default Libs;

View File

@@ -1,53 +0,0 @@
// executeScriptMV2 使用 chrome.tabs.executeScript
function build_code(fn, args) {
if (typeof fn === 'function') {
if (Array.isArray(args) && args.length) {
return `(${fn.toString()}).apply(null, ${JSON.stringify(args)});`;
}
return `(${fn.toString()})();`;
}
return fn;
}
// execute_script(tabId, fn, args?, runAt?)
export function execute_script(tab_id, fn, args, run_at) {
run_at = run_at || 'document_idle';
const code = build_code(fn, args);
return new Promise((resolve, reject) => {
chrome.tabs.executeScript(
tab_id,
{
code,
runAt: run_at,
},
(result) => {
if (chrome.runtime.lastError) {
return reject(new Error(chrome.runtime.lastError.message));
}
resolve(result);
},
);
});
}
export function inject_file(tab_id, file, run_at) {
run_at = run_at || 'document_idle';
return new Promise((resolve, reject) => {
chrome.tabs.executeScript(
tab_id,
{
file,
runAt: run_at,
},
() => {
if (chrome.runtime.lastError) {
return reject(new Error(chrome.runtime.lastError.message));
}
resolve(true);
},
);
});
}

View File

@@ -1,76 +1,365 @@
// openTabMV2 版本(极简 + 回调风格)
// tabs.jsMV2 Tab 操作工具Promise 风格)
import { execute_script } from './inject.js';
// ──────────── Chrome API Promise 封装 ────────────
function attach_tab_helpers(tab) {
function chrome_tabs_get(tab_id) {
return new Promise((resolve, reject) => {
chrome.tabs.get(tab_id, (t) => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(t);
});
});
}
function chrome_tabs_update(tab_id, props) {
return new Promise((resolve, reject) => {
chrome.tabs.update(tab_id, props, (t) => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(t || true);
});
});
}
function chrome_tabs_remove(tab_id) {
return new Promise((resolve, reject) => {
chrome.tabs.remove(tab_id, () => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(true);
});
});
}
function chrome_tabs_create(opts) {
return new Promise((resolve, reject) => {
chrome.tabs.create(opts, (t) => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(t);
});
});
}
function chrome_windows_create(opts) {
return new Promise((resolve, reject) => {
chrome.windows.create(opts, (w) => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(w);
});
});
}
function chrome_tabs_execute_script(tab_id, details) {
return new Promise((resolve, reject) => {
chrome.tabs.executeScript(tab_id, details, (result) => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(result);
});
});
}
/**
* 等待 tab 进入 status=complete含超时
* 先检查当前状态,已 complete 则直接返回
*/
function wait_tab_status_complete(tab_id, timeout_ms = 45000) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
chrome.tabs.onUpdated.removeListener(listener);
reject(new Error('等待页面加载超时'));
}, timeout_ms);
const listener = (id, info, tab) => {
if (id !== tab_id || info.status !== 'complete') return;
chrome.tabs.onUpdated.removeListener(listener);
clearTimeout(timer);
resolve(tab);
};
chrome.tabs.onUpdated.addListener(listener);
});
}
// ──────────── 代码构建 ────────────
const build_code = (fn, args) => {
if (typeof fn === 'function') {
const func_str = fn.toString();
if (Array.isArray(args) && args.length > 0) {
const serialized = JSON.stringify(args, (key, value) => {
if (typeof value === 'function') return undefined;
if (value && typeof value === 'object' && value.constructor === Object) {
try { JSON.stringify(value); return value; } catch { return '[Object]'; }
}
return value;
});
return `(${func_str}).apply(null, ${serialized});`;
}
return `(${func_str})();`;
}
if (typeof fn === 'string') return fn;
throw new TypeError('fn must be a function or string');
};
// ──────────── 脚本执行(低阶) ────────────
/**
* 在页面上下文执行脚本page world 桥接)
* 通过 CustomEvent + chrome.runtime.onMessage 回传结果
*/
export async function raw_execute_script(tab_id, fn, args = [], run_at = 'document_idle') {
const request_id = `${Date.now()}_${Math.random().toString().slice(2)}`;
const event_name = `__mv2_simple_page_exec_done__${request_id}`;
const page_exec_stmt = typeof fn === 'function'
? `__exec_result = ${build_code(fn, args)}`
: `__exec_result = (function () { ${fn} })();`;
const page_script_text = `
(function () {
const __request_id = ${JSON.stringify(request_id)};
const __event_name = ${JSON.stringify(event_name)};
let __exec_result;
Promise.resolve()
.then(() => {
${page_exec_stmt}
return __exec_result;
})
.then((__result) => {
window.dispatchEvent(new CustomEvent(__event_name, {
detail: { request_id: __request_id, ok: true, result: __result }
}));
})
.catch((__err) => {
window.dispatchEvent(new CustomEvent(__event_name, {
detail: {
request_id: __request_id,
ok: false,
error: {
message: (__err && __err.message) ? __err.message : String(__err),
stack: (__err && __err.stack) ? __err.stack : ''
}
}
}));
});
})();
`.trim();
const bootstrap_code = `
(function () {
const __request_id = ${JSON.stringify(request_id)};
const __event_name = ${JSON.stringify(event_name)};
const __on_done = (ev) => {
const detail = ev && ev.detail ? ev.detail : null;
if (!detail || detail.request_id !== __request_id) return;
window.removeEventListener(__event_name, __on_done, true);
try {
chrome.runtime.sendMessage({
channel: 'page_exec_bridge',
request_id: __request_id,
ok: !!detail.ok,
result: detail.result,
error_message: detail.error && detail.error.message ? detail.error.message : null,
error_stack: detail.error && detail.error.stack ? detail.error.stack : null
});
} catch (_) {}
};
window.addEventListener(__event_name, __on_done, true);
const el = document.createElement('script');
el.type = 'text/javascript';
el.textContent = ${JSON.stringify(page_script_text)};
(document.head || document.documentElement).appendChild(el);
el.parentNode && el.parentNode.removeChild(el);
})();
`.trim();
// 同时监听 onMessage 回传 + executeScript 报错,无法再简化
return await new Promise((resolve, reject) => {
const timeout_id = setTimeout(() => {
chrome.runtime.onMessage.removeListener(on_message);
reject(new Error(`Script execution timeout for tab ${tab_id}`));
}, 30000);
const on_message = (message) => {
if (!message || message.channel !== 'page_exec_bridge' || message.request_id !== request_id) return;
clearTimeout(timeout_id);
chrome.runtime.onMessage.removeListener(on_message);
if (message.ok) return resolve([message.result]);
const err = new Error(message.error_message || 'page script execution failed');
err.stack = message.error_stack || err.stack;
return reject(err);
};
chrome.runtime.onMessage.addListener(on_message);
chrome.tabs.executeScript(tab_id, { code: bootstrap_code, runAt: run_at }, () => {
if (chrome.runtime.lastError) {
clearTimeout(timeout_id);
chrome.runtime.onMessage.removeListener(on_message);
reject(new Error(chrome.runtime.lastError.message));
}
});
});
}
// ──────────── 注入文件 ────────────
export async function inject_file(tab_id, file, run_at = 'document_idle') {
await chrome_tabs_execute_script(tab_id, { file, runAt: run_at });
return true;
}
// ──────────── 确保 injected.js 已加载 ────────────
export async function ensure_injected(tab_id, max_retries = 3) {
// 先检查是否已注入
try {
const frames = await raw_execute_script(tab_id, () => !!window.__mv2_simple_injected, [], 'document_idle');
const injected = Array.isArray(frames) && frames.length ? (frames[0]?.result ?? frames[0]) : null;
if (injected === true) return true;
} catch (_) {
// 检查失败时继续尝试注入
}
let last_error;
for (let i = 1; i <= max_retries; i += 1) {
try {
await inject_file(tab_id, 'injected/injected.js', 'document_idle');
const frames = await raw_execute_script(tab_id, () => !!window.__mv2_simple_injected, [], 'document_idle');
const injected = Array.isArray(frames) && frames.length ? (frames[0]?.result ?? frames[0]) : null;
if (injected === true) return true;
if (i < max_retries) await new Promise((r) => setTimeout(r, 500 * i));
} catch (err) {
last_error = err;
if (i < max_retries) await new Promise((r) => setTimeout(r, 1000 * i));
}
}
throw new Error(`注入失败(重试 ${max_retries} 次): ${last_error?.message || 'unknown'}`);
}
// ──────────── 高阶脚本执行 ────────────
export async function execute_script(tab_id, fn, args = [], run_at = 'document_idle', options = {}) {
const opts = { ensure_injected: true, max_retries: 3, ...options };
if (opts.ensure_injected) {
await ensure_injected(tab_id, opts.max_retries);
}
return await raw_execute_script(tab_id, fn, args, run_at);
}
// ──────────── Tab 辅助方法绑定 ────────────
const attach_tab_helpers = (tab) => {
if (!tab) return tab;
tab.remove = function remove(delay_ms) {
delay_ms = Number.isFinite(delay_ms) ? delay_ms : 0;
setTimeout(() => {
tab.off_update_complete && tab.off_update_complete();
chrome.tabs.remove(tab.id, () => void 0);
}, Math.max(0, delay_ms));
};
tab.execute_script = async function execute_script_on_tab(fn, args, run_at) {
return await execute_script(tab.id, fn, args, run_at);
tab.execute_script = (fn, args, run_at) => execute_script(tab.id, fn, args, run_at);
tab.inject_file = (file, run_at) => inject_file(tab.id, file, run_at);
tab.ensure_injected = () => ensure_injected(tab.id);
tab.navigate = async (url, options) => {
const active = options && options.active === true;
return await chrome_tabs_update(tab.id, { url: String(url), active });
};
tab.wait_complete = async function wait_complete(timeout_ms) {
const timeout = Number.isFinite(timeout_ms) ? Math.max(0, timeout_ms) : 45000;
const t0 = await chrome_tabs_get(tab.id).catch(() => null);
if (t0 && t0.status === 'complete') return t0;
return await wait_tab_status_complete(tab.id, timeout);
};
tab._on_update_complete_listener = null;
tab.on_update_complete = function on_update_complete(fn, options) {
if (typeof fn !== 'function' || !tab.id) return false;
tab.off_update_complete && tab.off_update_complete();
let running = false;
const once = !!(options && options.once === true);
const listener = async (updated_tab_id, change_info, updated_tab) => {
if (updated_tab_id !== tab.id || !change_info || change_info.status !== 'complete') return;
if (running) return;
running = true;
const tab_obj = attach_tab_helpers(updated_tab || tab);
await fn(tab_obj, change_info);
if (once) tab.off_update_complete && tab.off_update_complete();
running = false;
};
chrome.tabs.onUpdated.addListener(listener);
tab._on_update_complete_listener = listener;
// 注册时如果已 complete立即触发一次
chrome.tabs.get(tab.id, (t0) => {
if (chrome.runtime.lastError) return;
if (t0 && t0.status === 'complete') {
listener(tab.id, { status: 'complete' }, t0);
}
});
return true;
};
tab.wait_update_complete_once = function wait_update_complete_once(worker) {
return new Promise((resolve, reject) => {
tab.on_update_complete(async () => {
try {
resolve(await worker(tab));
} catch (err) {
reject(err);
}
}, { once: true });
});
};
tab.off_update_complete = function off_update_complete() {
if (!tab._on_update_complete_listener) return;
try { chrome.tabs.onUpdated.removeListener(tab._on_update_complete_listener); } catch (_) {}
tab._on_update_complete_listener = null;
};
tab.close_window = function close_window(delay_ms) {
delay_ms = Number.isFinite(delay_ms) ? delay_ms : 0;
setTimeout(() => {
if (tab.windowId) {
tab.off_update_complete && tab.off_update_complete();
chrome.windows.remove(tab.windowId, () => void 0);
}
}, Math.max(0, delay_ms));
};
return tab;
}
};
export function open_tab(url, options) {
// 保留原本 Promise 版本(内部复用)
options = options && typeof options === 'object' ? options : {};
// ──────────── 打开标签页 ────────────
return new Promise((resolve, reject) => {
chrome.tabs.create(
{
url: 'about:blank',
active: options.active !== false,
},
(tab) => {
if (chrome.runtime.lastError) {
return reject(new Error(chrome.runtime.lastError.message));
}
if (!tab || !tab.id) {
return reject(new Error('tab 创建失败'));
}
const tab_id = tab.id;
const on_updated = (updated_tab_id, change_info, updated_tab) => {
if (updated_tab_id !== tab_id) return;
if (change_info.status !== 'complete') return;
chrome.tabs.onUpdated.removeListener(on_updated);
resolve({ tab_id, tab: attach_tab_helpers(updated_tab) });
};
chrome.tabs.onUpdated.addListener(on_updated);
chrome.tabs.update(tab_id, { url });
},
);
export async function open_tab(url, options = {}) {
const opts = { active: true, timeout: 45000, loadInBackground: false, ...options };
const tab = await chrome_tabs_create({
url: 'about:blank',
active: !opts.loadInBackground && opts.active,
});
if (!tab || !tab.id) throw new Error('创建标签页失败');
await chrome_tabs_update(tab.id, { url });
const done_tab = await wait_tab_status_complete(tab.id, opts.timeout);
return { tab_id: tab.id, tab: attach_tab_helpers(done_tab) };
}
export function close_tab(tab_id, delay_ms) {
delay_ms = Number.isFinite(delay_ms) ? delay_ms : 0;
setTimeout(() => {
chrome.tabs.remove(tab_id, () => void 0);
}, Math.max(0, delay_ms));
// ──────────── 关闭标签页 ────────────
export async function close_tab(tab_id, delay_ms = 0) {
if (delay_ms > 0) {
await new Promise((r) => setTimeout(r, delay_ms));
}
return await chrome_tabs_remove(tab_id);
}
// openTab 任务对象:用对象绑定方法,减少重复参数
// ──────────── Tab 任务对象 ────────────
export function create_tab_task(url) {
const task = {
url,
@@ -81,74 +370,38 @@ export function create_tab_task(url) {
height: 900,
target: null,
active: true,
// 你期望的写法tab_task.on_updated = () => {}
on_error: null,
on_updated: null,
set_bounds(bounds) {
bounds = bounds && typeof bounds === 'object' ? bounds : {};
if (Object.prototype.hasOwnProperty.call(bounds, 'top')) this.top = bounds.top;
if (Object.prototype.hasOwnProperty.call(bounds, 'left')) this.left = bounds.left;
if (Object.prototype.hasOwnProperty.call(bounds, 'width')) this.width = bounds.width;
if (Object.prototype.hasOwnProperty.call(bounds, 'height')) this.height = bounds.height;
return this;
},
set_target(target) {
this.target = target || null;
return this;
},
set_latest(latest) {
this.latest = !!latest;
return this;
},
set_active(active) {
this.active = active !== false;
if ('top' in bounds) this.top = bounds.top;
if ('left' in bounds) this.left = bounds.left;
if ('width' in bounds) this.width = bounds.width;
if ('height' in bounds) this.height = bounds.height;
return this;
},
set_target(target) { this.target = target || null; return this; },
set_latest(latest) { this.latest = !!latest; return this; },
set_active(active) { this.active = active !== false; return this; },
async open_async() {
// 用 chrome.windows.create 新开窗口承载 tab
const win = await new Promise((resolve, reject) => {
chrome.windows.create(
{
url: 'about:blank',
type: 'popup',
focused: true,
top: this.top,
left: this.left,
width: this.width,
height: this.height,
},
(w) => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(w);
},
);
const win = await chrome_windows_create({
url: 'about:blank',
type: 'popup',
focused: true,
top: this.top,
left: this.left,
width: this.width,
height: this.height,
});
const tab0 = win && win.tabs && win.tabs[0] ? win.tabs[0] : null;
if (!tab0 || !tab0.id) {
throw new Error('popup window 创建失败');
}
if (!tab0 || !tab0.id) throw new Error('popup window 创建失败');
await new Promise((resolve, reject) => {
chrome.tabs.update(tab0.id, { url: this.url, active: this.active !== false }, () => {
if (chrome.runtime.lastError) return reject(new Error(chrome.runtime.lastError.message));
resolve(true);
});
});
const tab_done = await new Promise((resolve) => {
const on_updated = (tab_id, change_info, tab) => {
if (tab_id !== tab0.id) return;
if (change_info.status !== 'complete') return;
chrome.tabs.onUpdated.removeListener(on_updated);
resolve(tab);
};
chrome.tabs.onUpdated.addListener(on_updated);
});
return attach_tab_helpers(tab_done);
await chrome_tabs_update(tab0.id, { url: this.url, active: this.active !== false });
const done_tab = await wait_tab_status_complete(tab0.id);
return attach_tab_helpers(done_tab);
},
};
return task;
}

View File

@@ -5,6 +5,7 @@
"description": "MV2 极简骨架openTab + executeScript + __REQUEST_DONE 监听",
"permissions": [
"tabs",
"storage",
"<all_urls>"
],
"background": {
@@ -20,7 +21,8 @@
}
],
"web_accessible_resources": [
"content/request_watcher.js"
"content/request_watcher.js",
"injected/injected.js"
],
"browser_action": {
"default_title": "mv2_simple_crx"

View File

@@ -1,9 +1,9 @@
const out = document.getElementById('out');
const btn = document.getElementById('btn');
function set_out(obj) {
const set_out = (obj) => {
out.textContent = typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2);
}
};
btn.addEventListener('click', () => {
set_out('执行中...');

View File

@@ -83,6 +83,22 @@ body {
font-size: 12px;
}
.label_row {
display: flex;
align-items: flex-start;
gap: 8px;
margin-top: 10px;
font-size: 12px;
color: var(--text);
line-height: 1.5;
cursor: pointer;
}
.label_row input {
margin-top: 3px;
flex-shrink: 0;
}
.input,
.textarea {
width: 100%;
@@ -159,6 +175,15 @@ body {
overflow: auto;
}
/* 响应区:不显示滚动条,长行自动换行(含无空格长串) */
.pre_response {
overflow: visible;
overflow-x: hidden;
white-space: pre-wrap;
overflow-wrap: anywhere;
word-break: break-word;
}
.pre_scroll {
max-height: 520px;
overflow: auto;

View File

@@ -20,7 +20,12 @@
<div class="card_title">调用</div>
<div class="form">
<div>
<button id="btn_bg_reload" class="btn">刷新后台</button>
</div>
<label class="label">方法名action</label>
<!-- action 列表:由 background 注册;这里仅提供快速手动调用入口 -->
<select id="action_name" class="input">
<option value="zhipu_query_position_page">zhipu_query_position_page</option>
<option value="amazon_top_list">amazon_top_list</option>
@@ -28,16 +33,19 @@
<option value="amazon_set_language">amazon_set_language</option>
<option value="amazon_product_detail">amazon_product_detail</option>
<option value="amazon_product_reviews">amazon_product_reviews</option>
<option value="amazon_product_detail_reviews">amazon_product_detail_reviews</option>
</select>
<label class="label">参数JSON</label>
<div id="action_params_desc" class="hint" style="margin-top:6px; white-space:pre-wrap;"></div>
<textarea id="action_params" class="textarea" spellcheck="false">{}</textarea>
<label class="label_row"><input type="checkbox" id="opt_keep_tab_open" checked /> 执行后保留自动化窗口keep_tab_open覆盖下方 JSON 与本 action 默认值)</label>
<div class="row">
<button id="btn_run" class="btn primary">执行</button>
<button id="btn_clear" class="btn">清空日志</button>
<button id="btn_bg_reload" class="btn">刷新后台</button>
</div>
<div class="hint">
@@ -52,7 +60,7 @@
<label class="label">动作日志</label>
<pre id="action_log" class="pre pre_small"></pre>
<div class="card_title">响应</div>
<pre id="last_response" class="pre"></pre>
<pre id="last_response" class="pre pre_response"></pre>
</div>
</div>
</div>

View File

@@ -6,24 +6,25 @@ const btn_clear_el = document.getElementById('btn_clear');
const btn_bg_reload_el = document.getElementById('btn_bg_reload');
const last_response_el = document.getElementById('last_response');
const action_log_el = document.getElementById('action_log');
const opt_keep_tab_open_el = document.getElementById('opt_keep_tab_open');
let actions_meta = {};
const ui_state = { last_result: null, actions: [] };
function now_time() {
const now_time = () => {
const d = new Date();
const pad = (n) => String(n).padStart(2, '0');
return `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
}
};
function safe_json_parse(text) {
const safe_json_parse = (text) => {
try {
return JSON.parse(text);
} catch (e) {
return { __parse_error: e.message, __raw: text };
}
}
};
function pick_main_result(res) {
const pick_main_result = (res) => {
// 右侧只展示最核心的数据,避免被 ok/request_id 包裹层干扰
if (res && res.ok && res.data) {
// 约定action 返回的核心结果放在 data.result例如 amazon_search_list 的 stage=list
@@ -31,14 +32,14 @@ function pick_main_result(res) {
return res.data;
}
return res;
}
};
function render_state() {
const render_state = () => {
last_response_el.textContent = JSON.stringify(ui_state.last_result, null, 2);
action_log_el.textContent = ui_state.actions.join('\n');
}
};
function push_action(obj) {
const push_action = (obj) => {
// 动作日志只保留单行文本,避免 JSON 换行太长
const ts = now_time();
const type = obj && obj.type ? String(obj.type) : 'action';
@@ -64,11 +65,16 @@ function push_action(obj) {
ui_state.actions.splice(0, ui_state.actions.length - 200);
}
render_state();
}
};
const apply_keep_tab_open_override = (parsed) => {
if (!opt_keep_tab_open_el || !parsed || typeof parsed !== 'object' || parsed.__parse_error) return parsed;
return { ...parsed, keep_tab_open: opt_keep_tab_open_el.checked === true };
};
btn_run_el.addEventListener('click', () => {
const action = action_name_el.value;
const params = safe_json_parse(action_params_el.value || '{}');
const params = apply_keep_tab_open_override(safe_json_parse(action_params_el.value || '{}'));
push_action({ type: 'call', action, params });
ui_state.last_result = { running: true, action, params };

6
package-lock.json generated Normal file
View File

@@ -0,0 +1,6 @@
{
"name": "mv2_simple_crx",
"lockfileVersion": 3,
"requires": true,
"packages": {}
}

View File

@@ -21,8 +21,8 @@ const port = cfg.server.port;
await sequelize.authenticate();
// await sequelize.sync();
start_all_cron_tasks();
await start_all_cron_tasks();
app.listen(port);
// eslint-disable-next-line no-console
console.log(`server listening on ${port}`);
console.log(`[${new Date().toLocaleString()}] server listening on ${port}`);

View File

@@ -56,7 +56,11 @@ export function get_app_config() {
action_timeout_ms: get_int('ACTION_TIMEOUT_MS', 300000),
puppeteer_headless: get_bool('PUPPETEER_HEADLESS', false),
chrome_executable_path: (get_env('CHROME_EXECUTABLE_PATH') || '').trim() || path.resolve(__dirname, '../../chrome-win/chrome.exe'),
log_invoke_action: get_bool('LOG_INVOKE_ACTION', true)
log_invoke_action: get_bool('LOG_INVOKE_ACTION', true),
auto_close_browser: get_bool('AUTO_CLOSE_BROWSER', true),
enable_stealth: get_bool('ENABLE_STEALTH', true),
log_sql: get_bool('LOG_SQL', false),
log_sql_benchmark: get_bool('LOG_SQL_BENCHMARK', false)
}
};

View File

@@ -4,11 +4,21 @@
*/
export const cron_task_list = [
// 示例:每 6 小时跑一次列表抓取
// 任务流:先跑列表,再依赖列表 URL 跑“详情+评论(合并 action
{
name: 'amazon_search_list_every_6h',
cron_expression: '0 */1 * * *',
action_name: 'amazon_search_list',
action_payload: { keyword: '野餐包', limit: 100 }
name: 'amazon_search_detail_reviews_every_1h',
cron_expression: '0 */1 * * *', // 1小时执行一次
type: 'flow',
flow_name: 'amazon_search_detail_reviews',
flow_payload: {
// 插件参数category_keyword / sort_by / limit
category_keyword: '野餐包',
// featured / price_asc / price_desc / review / newest / bestseller
sort_by: 'bestseller',
limit: 100,
// flow 自己的参数
reviews_limit: 50,
gap_ms: 500
}
}
];

View File

@@ -2,7 +2,6 @@ import { get_app_config } from './app_config.js';
export function get_sequelize_options() {
const cfg = get_app_config();
console.log( 'get_sequelize_options', cfg.mysql );
return {
host: cfg.mysql.host,
@@ -11,7 +10,18 @@ export function get_sequelize_options() {
password: cfg.mysql.password,
database: cfg.mysql.database,
dialect: 'mysql',
logging: false,
benchmark: cfg.crawler.log_sql_benchmark === true,
logging: cfg.crawler.log_sql === true
? (sql, timing_ms) => {
if (cfg.crawler.log_sql_benchmark === true && typeof timing_ms === 'number') {
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] [sql]`, { timing_ms, sql });
return;
}
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] [sql]`, sql);
}
: false,
define: {
underscored: true,
timestamps: true,

View File

@@ -6,26 +6,24 @@ export function define_amazon_product(sequelize) {
{
id: { type: DataTypes.BIGINT.UNSIGNED, primaryKey: true, autoIncrement: true },
asin: { type: DataTypes.STRING(32), allowNull: false },
url: { type: DataTypes.TEXT, allowNull: false },
title: { type: DataTypes.TEXT, allowNull: true },
url: { type: DataTypes.STRING(2048), allowNull: false },
title: { type: DataTypes.STRING(1024), allowNull: true },
price: { type: DataTypes.STRING(64), allowNull: true },
sku: { type: DataTypes.STRING(256), allowNull: true },
sku_color: { type: DataTypes.STRING(128), allowNull: true },
sku_size: { type: DataTypes.STRING(128), allowNull: true },
brand_line: { type: DataTypes.TEXT, allowNull: true },
brand_store_url: { type: DataTypes.TEXT, allowNull: true },
sku_json: { type: DataTypes.JSON, allowNull: true, comment: 'sku 结构化 JSON如 {color:[], size:[]}' },
brand_line: { type: DataTypes.STRING(512), allowNull: true },
brand_store_url: { type: DataTypes.STRING(2048), allowNull: true },
ac_badge: { type: DataTypes.STRING(128), allowNull: true },
bestseller_hint: { type: DataTypes.TEXT, allowNull: true },
delivery_hint: { type: DataTypes.TEXT, allowNull: true },
social_proof: { type: DataTypes.TEXT, allowNull: true },
sustainability_hint: { type: DataTypes.TEXT, allowNull: true },
bestseller_hint: { type: DataTypes.STRING(512), allowNull: true },
delivery_hint: { type: DataTypes.STRING(512), allowNull: true },
social_proof: { type: DataTypes.STRING(256), allowNull: true },
sustainability_hint: { type: DataTypes.STRING(256), allowNull: true },
rating_stars: { type: DataTypes.STRING(64), allowNull: true },
review_count_text: { type: DataTypes.STRING(64), allowNull: true },
main_image: { type: DataTypes.TEXT, allowNull: true },
images_json: { type: DataTypes.TEXT('long'), allowNull: true },
bullets_json: { type: DataTypes.TEXT('long'), allowNull: true },
product_info_json: { type: DataTypes.TEXT('long'), allowNull: true },
detail_extra_lines_json: { type: DataTypes.TEXT('long'), allowNull: true }
main_image: { type: DataTypes.STRING(2048), allowNull: true },
images_json: { type: DataTypes.JSON, allowNull: true },
bullets_json: { type: DataTypes.JSON, allowNull: true },
product_info_json: { type: DataTypes.JSON, allowNull: true },
detail_extra_lines_json: { type: DataTypes.JSON, allowNull: true }
},
{
tableName: 'amazon_product',

View File

@@ -6,10 +6,10 @@ export function define_amazon_review(sequelize) {
{
id: { type: DataTypes.BIGINT.UNSIGNED, primaryKey: true, autoIncrement: true },
asin: { type: DataTypes.STRING(32), allowNull: true },
url: { type: DataTypes.TEXT, allowNull: false },
url: { type: DataTypes.STRING(2048), allowNull: false },
review_id: { type: DataTypes.STRING(64), allowNull: false },
author: { type: DataTypes.STRING(256), allowNull: true },
title: { type: DataTypes.TEXT, allowNull: true },
title: { type: DataTypes.STRING(512), allowNull: true },
body: { type: DataTypes.TEXT('long'), allowNull: true },
rating_text: { type: DataTypes.STRING(64), allowNull: true },
review_date: { type: DataTypes.STRING(128), allowNull: true },

View File

@@ -6,8 +6,8 @@ export function define_amazon_search_item(sequelize) {
{
id: { type: DataTypes.BIGINT.UNSIGNED, primaryKey: true, autoIncrement: true },
asin: { type: DataTypes.STRING(32), allowNull: false },
url: { type: DataTypes.TEXT, allowNull: false },
title: { type: DataTypes.TEXT, allowNull: true },
url: { type: DataTypes.STRING(2048), allowNull: false },
title: { type: DataTypes.STRING(1024), allowNull: true },
price: { type: DataTypes.STRING(64), allowNull: true },
rating: { type: DataTypes.FLOAT, allowNull: true },
rating_text: { type: DataTypes.STRING(64), allowNull: true },

460
server/package-lock.json generated
View File

@@ -14,6 +14,8 @@
"mysql2": "^3.11.0",
"node-cron": "^3.0.3",
"puppeteer": "^23.4.1",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"sequelize": "^6.37.3"
}
},
@@ -182,6 +184,15 @@
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"license": "Python-2.0"
},
"node_modules/arr-union": {
"version": "3.1.0",
"resolved": "https://registry.npmmirror.com/arr-union/-/arr-union-3.1.0.tgz",
"integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/ast-types": {
"version": "0.13.4",
"resolved": "https://registry.npmmirror.com/ast-types/-/ast-types-0.13.4.tgz",
@@ -217,6 +228,12 @@
}
}
},
"node_modules/balanced-match": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
"license": "MIT"
},
"node_modules/bare-events": {
"version": "2.8.2",
"resolved": "https://registry.npmmirror.com/bare-events/-/bare-events-2.8.2.tgz",
@@ -333,6 +350,16 @@
"node": ">=10.0.0"
}
},
"node_modules/brace-expansion": {
"version": "1.1.12",
"resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.12.tgz",
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz",
@@ -453,6 +480,22 @@
"node": ">=12"
}
},
"node_modules/clone-deep": {
"version": "0.2.4",
"resolved": "https://registry.npmmirror.com/clone-deep/-/clone-deep-0.2.4.tgz",
"integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==",
"license": "MIT",
"dependencies": {
"for-own": "^0.1.3",
"is-plain-object": "^2.0.1",
"kind-of": "^3.0.2",
"lazy-cache": "^1.0.3",
"shallow-clone": "^0.1.2"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/co": {
"version": "4.6.0",
"resolved": "https://registry.npmmirror.com/co/-/co-4.6.0.tgz",
@@ -497,6 +540,12 @@
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"license": "MIT"
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
"license": "MIT"
},
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -595,6 +644,15 @@
"integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==",
"license": "MIT"
},
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmmirror.com/deepmerge/-/deepmerge-4.3.1.tgz",
"integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/degenerator": {
"version": "5.0.1",
"resolved": "https://registry.npmmirror.com/degenerator/-/degenerator-5.0.1.tgz",
@@ -858,6 +916,27 @@
"pend": "~1.2.0"
}
},
"node_modules/for-in": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/for-in/-/for-in-1.0.2.tgz",
"integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/for-own": {
"version": "0.1.5",
"resolved": "https://registry.npmmirror.com/for-own/-/for-own-0.1.5.tgz",
"integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==",
"license": "MIT",
"dependencies": {
"for-in": "^1.0.1"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz",
@@ -867,6 +946,26 @@
"node": ">= 0.6"
}
},
"node_modules/fs-extra": {
"version": "10.1.0",
"resolved": "https://registry.npmmirror.com/fs-extra/-/fs-extra-10.1.0.tgz",
"integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.2.0",
"jsonfile": "^6.0.1",
"universalify": "^2.0.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
"license": "ISC"
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz",
@@ -969,6 +1068,27 @@
"node": ">= 14"
}
},
"node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.3.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
"deprecated": "Glob versions prior to v9 are no longer supported",
"license": "ISC",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.1.1",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmmirror.com/gopd/-/gopd-1.2.0.tgz",
@@ -981,6 +1101,12 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/graceful-fs": {
"version": "4.2.11",
"resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.11.tgz",
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
"license": "ISC"
},
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.1.0.tgz",
@@ -1183,6 +1309,17 @@
],
"license": "MIT"
},
"node_modules/inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
"deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
"license": "ISC",
"dependencies": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz",
@@ -1204,6 +1341,21 @@
"integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
"license": "MIT"
},
"node_modules/is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmmirror.com/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
"license": "MIT"
},
"node_modules/is-extendable": {
"version": "0.1.1",
"resolved": "https://registry.npmmirror.com/is-extendable/-/is-extendable-0.1.1.tgz",
"integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -1232,6 +1384,18 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/is-plain-object": {
"version": "2.0.4",
"resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-2.0.4.tgz",
"integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
"license": "MIT",
"dependencies": {
"isobject": "^3.0.1"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/is-property": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/is-property/-/is-property-1.0.2.tgz",
@@ -1256,6 +1420,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/isobject": {
"version": "3.0.1",
"resolved": "https://registry.npmmirror.com/isobject/-/isobject-3.0.1.tgz",
"integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmmirror.com/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -1280,6 +1453,18 @@
"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
"license": "MIT"
},
"node_modules/jsonfile": {
"version": "6.2.0",
"resolved": "https://registry.npmmirror.com/jsonfile/-/jsonfile-6.2.0.tgz",
"integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==",
"license": "MIT",
"dependencies": {
"universalify": "^2.0.0"
},
"optionalDependencies": {
"graceful-fs": "^4.1.6"
}
},
"node_modules/keygrip": {
"version": "1.1.0",
"resolved": "https://registry.npmmirror.com/keygrip/-/keygrip-1.1.0.tgz",
@@ -1293,6 +1478,18 @@
"node": ">= 0.6"
}
},
"node_modules/kind-of": {
"version": "3.2.2",
"resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-3.2.2.tgz",
"integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
"license": "MIT",
"dependencies": {
"is-buffer": "^1.1.5"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/koa": {
"version": "2.16.4",
"resolved": "https://registry.npmmirror.com/koa/-/koa-2.16.4.tgz",
@@ -1394,6 +1591,15 @@
"node": ">= 0.6"
}
},
"node_modules/lazy-cache": {
"version": "1.0.4",
"resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-1.0.4.tgz",
"integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/lines-and-columns": {
"version": "1.2.4",
"resolved": "https://registry.npmmirror.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@@ -1454,6 +1660,20 @@
"node": ">= 0.6"
}
},
"node_modules/merge-deep": {
"version": "3.0.3",
"resolved": "https://registry.npmmirror.com/merge-deep/-/merge-deep-3.0.3.tgz",
"integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
"license": "MIT",
"dependencies": {
"arr-union": "^3.1.0",
"clone-deep": "^0.2.4",
"kind-of": "^3.0.2"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/methods": {
"version": "1.1.2",
"resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz",
@@ -1484,12 +1704,46 @@
"node": ">= 0.6"
}
},
"node_modules/minimatch": {
"version": "3.1.5",
"resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.5.tgz",
"integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
"license": "ISC",
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/mitt": {
"version": "3.0.1",
"resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
"license": "MIT"
},
"node_modules/mixin-object": {
"version": "2.0.1",
"resolved": "https://registry.npmmirror.com/mixin-object/-/mixin-object-2.0.1.tgz",
"integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==",
"license": "MIT",
"dependencies": {
"for-in": "^0.1.3",
"is-extendable": "^0.1.1"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/mixin-object/node_modules/for-in": {
"version": "0.1.8",
"resolved": "https://registry.npmmirror.com/for-in/-/for-in-0.1.8.tgz",
"integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/moment": {
"version": "2.30.1",
"resolved": "https://registry.npmmirror.com/moment/-/moment-2.30.1.tgz",
@@ -1690,6 +1944,15 @@
"node": ">= 0.8"
}
},
"node_modules/path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/path-to-regexp": {
"version": "6.3.0",
"resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
@@ -1797,6 +2060,142 @@
"node": ">=18"
}
},
"node_modules/puppeteer-extra": {
"version": "3.3.6",
"resolved": "https://registry.npmmirror.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz",
"integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==",
"license": "MIT",
"dependencies": {
"@types/debug": "^4.1.0",
"debug": "^4.1.1",
"deepmerge": "^4.2.2"
},
"engines": {
"node": ">=8"
},
"peerDependencies": {
"@types/puppeteer": "*",
"puppeteer": "*",
"puppeteer-core": "*"
},
"peerDependenciesMeta": {
"@types/puppeteer": {
"optional": true
},
"puppeteer": {
"optional": true
},
"puppeteer-core": {
"optional": true
}
}
},
"node_modules/puppeteer-extra-plugin": {
"version": "3.2.3",
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz",
"integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==",
"license": "MIT",
"dependencies": {
"@types/debug": "^4.1.0",
"debug": "^4.1.1",
"merge-deep": "^3.0.1"
},
"engines": {
"node": ">=9.11.2"
},
"peerDependencies": {
"playwright-extra": "*",
"puppeteer-extra": "*"
},
"peerDependenciesMeta": {
"playwright-extra": {
"optional": true
},
"puppeteer-extra": {
"optional": true
}
}
},
"node_modules/puppeteer-extra-plugin-stealth": {
"version": "2.11.2",
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz",
"integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==",
"license": "MIT",
"dependencies": {
"debug": "^4.1.1",
"puppeteer-extra-plugin": "^3.2.3",
"puppeteer-extra-plugin-user-preferences": "^2.4.1"
},
"engines": {
"node": ">=8"
},
"peerDependencies": {
"playwright-extra": "*",
"puppeteer-extra": "*"
},
"peerDependenciesMeta": {
"playwright-extra": {
"optional": true
},
"puppeteer-extra": {
"optional": true
}
}
},
"node_modules/puppeteer-extra-plugin-user-data-dir": {
"version": "2.4.1",
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz",
"integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==",
"license": "MIT",
"dependencies": {
"debug": "^4.1.1",
"fs-extra": "^10.0.0",
"puppeteer-extra-plugin": "^3.2.3",
"rimraf": "^3.0.2"
},
"engines": {
"node": ">=8"
},
"peerDependencies": {
"playwright-extra": "*",
"puppeteer-extra": "*"
},
"peerDependenciesMeta": {
"playwright-extra": {
"optional": true
},
"puppeteer-extra": {
"optional": true
}
}
},
"node_modules/puppeteer-extra-plugin-user-preferences": {
"version": "2.4.1",
"resolved": "https://registry.npmmirror.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz",
"integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==",
"license": "MIT",
"dependencies": {
"debug": "^4.1.1",
"deepmerge": "^4.2.2",
"puppeteer-extra-plugin": "^3.2.3",
"puppeteer-extra-plugin-user-data-dir": "^2.4.1"
},
"engines": {
"node": ">=8"
},
"peerDependencies": {
"playwright-extra": "*",
"puppeteer-extra": "*"
},
"peerDependenciesMeta": {
"playwright-extra": {
"optional": true
},
"puppeteer-extra": {
"optional": true
}
}
},
"node_modules/qs": {
"version": "6.15.0",
"resolved": "https://registry.npmmirror.com/qs/-/qs-6.15.0.tgz",
@@ -1863,6 +2262,22 @@
"integrity": "sha512-hMD7odLOt3LkTjcif8aRZqi/hybjpLNgSk5oF5FCowfCjok6LukpN2bDX7R5wDmbgBQFn7YoBxSagmtXHaJYJw==",
"license": "MIT"
},
"node_modules/rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz",
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
"deprecated": "Rimraf versions prior to v4 are no longer supported",
"license": "ISC",
"dependencies": {
"glob": "^7.1.3"
},
"bin": {
"rimraf": "bin.js"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -1995,6 +2410,42 @@
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
"license": "ISC"
},
"node_modules/shallow-clone": {
"version": "0.1.2",
"resolved": "https://registry.npmmirror.com/shallow-clone/-/shallow-clone-0.1.2.tgz",
"integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==",
"license": "MIT",
"dependencies": {
"is-extendable": "^0.1.1",
"kind-of": "^2.0.1",
"lazy-cache": "^0.2.3",
"mixin-object": "^2.0.1"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/shallow-clone/node_modules/kind-of": {
"version": "2.0.1",
"resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-2.0.1.tgz",
"integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==",
"license": "MIT",
"dependencies": {
"is-buffer": "^1.0.2"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/shallow-clone/node_modules/lazy-cache": {
"version": "0.2.7",
"resolved": "https://registry.npmmirror.com/lazy-cache/-/lazy-cache-0.2.7.tgz",
"integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/side-channel": {
"version": "1.1.0",
"resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.1.0.tgz",
@@ -2291,6 +2742,15 @@
"integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
"license": "MIT"
},
"node_modules/universalify": {
"version": "2.0.1",
"resolved": "https://registry.npmmirror.com/universalify/-/universalify-2.0.1.tgz",
"integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
"license": "MIT",
"engines": {
"node": ">= 10.0.0"
}
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz",

View File

@@ -14,6 +14,8 @@
"mysql2": "^3.11.0",
"node-cron": "^3.0.3",
"puppeteer": "^23.4.1",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"sequelize": "^6.37.3"
}
}

2080
server/pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -2,5 +2,5 @@ import { sequelize } from '../models/index.js';
await sequelize.sync({ alter: true });
// eslint-disable-next-line no-console
console.log('sync ok');
console.log(`[${new Date().toLocaleString()}] sync ok`);
await sequelize.close();

View File

@@ -1,110 +0,0 @@
import {
amazon_product,
amazon_search_item,
amazon_review
} from '../models/index.js';
import { safe_json_stringify } from './json_utils.js';
function build_batch_key(prefix) {
return `${prefix}_${Date.now()}_${Math.random().toString().slice(2, 8)}`;
}
function pick_asin_from_url(url) {
if (!url) return null;
const m = String(url).match(/\/dp\/([A-Z0-9]{8,16})/i);
return m && m[1] ? m[1].toUpperCase() : null;
}
export async function persist_amazon_result(result) {
if (!result || !result.stage) {
return;
}
if (result.stage === 'detail') {
const asin = result.asin || pick_asin_from_url(result.url);
if (!asin) {
return;
}
await amazon_product.upsert({
asin,
url: result.url || '',
title: result.title || null,
price: result.price || null,
sku: result.sku || null,
sku_color: result.sku_color || null,
sku_size: result.sku_size || null,
brand_line: result.brand_line || null,
brand_store_url: result.brand_store_url || null,
ac_badge: result.ac_badge || null,
bestseller_hint: result.bestseller_hint || null,
delivery_hint: result.delivery_hint || null,
social_proof: result.social_proof || null,
sustainability_hint: result.sustainability_hint || null,
rating_stars: result.rating_stars || null,
review_count_text: result.review_count_text || null,
main_image: result.main_image || null,
images_json: safe_json_stringify(result.images || []),
bullets_json: safe_json_stringify(result.bullets || []),
product_info_json: safe_json_stringify(result.product_info || {}),
detail_extra_lines_json: safe_json_stringify(result.detail_extra_lines || [])
});
return;
}
if (result.stage === 'list') {
const batch_key = build_batch_key('list');
const items = Array.isArray(result.items) ? result.items : [];
for (const it of items) {
const asin = it.asin || pick_asin_from_url(it.url);
if (!asin || !it.url) continue;
await amazon_search_item.create({
asin,
url: it.url,
title: it.title || null,
price: it.price || null,
rating: typeof it.rating === 'number' ? it.rating : null,
rating_text: it.rating_text || null,
review_count: typeof it.review_count === 'number' ? it.review_count : null,
review_count_text: it.review_count_text || null,
rank_index: typeof it.index === 'number' ? it.index : null,
batch_key,
batch_total: typeof result.total === 'number' ? result.total : null,
batch_limit: typeof result.limit === 'number' ? result.limit : null
});
}
return;
}
if (result.stage === 'reviews') {
const batch_key = build_batch_key('reviews');
const asin = pick_asin_from_url(result.url);
const items = Array.isArray(result.items) ? result.items : [];
for (const it of items) {
const review_id = it.review_id;
if (!review_id) continue;
const asin_value = asin || pick_asin_from_url(it.url) || pick_asin_from_url(result.url);
await amazon_review.upsert({
asin: asin_value || null,
url: result.url || '',
review_id,
author: it.author || null,
title: it.title || null,
body: it.body || null,
rating_text: it.rating_text || null,
review_date: it.date || null,
review_index: typeof it.index === 'number' ? it.index : null,
batch_key,
batch_total: typeof result.total === 'number' ? result.total : null,
batch_limit: typeof result.limit === 'number' ? result.limit : null
});
}
}
}

View File

@@ -0,0 +1,262 @@
import { execute_action_and_record } from '../../task_executor.js';
import { sleep_ms } from '../flow_utils.js';
import { amazon_product, amazon_search_item, amazon_review } from '../../../models/index.js';
import { safe_json_stringify } from '../../json_utils.js';
import { close_browser } from '../../puppeteer/puppeteer_runner.js';
import { Op } from 'sequelize';
function build_batch_key(prefix) {
return `${prefix}_${Date.now()}_${Math.random().toString().slice(2, 8)}`;
}
function pick_asin_from_url(url) {
if (!url) return null;
const m = String(url).match(/\/dp\/([A-Z0-9]{8,16})/i);
return m && m[1] ? m[1].toUpperCase() : null;
}
/**
* 以“自然日(本地时区)”为口径判断当天是否已抓取过详情。
* - 详情数据写入 `amazon_product`,会更新 `updated_at`
* - 当 `updated_at >= 今日 00:00` 时,后续同日任务将跳过详情提取(仅抓评论)
*/
function get_today_start() {
const d = new Date();
d.setHours(0, 0, 0, 0);
return d;
}
async function has_detail_fetched_today(asin) {
if (!asin) return false;
const row = await amazon_product.findOne({
attributes: ['asin', 'updated_at'],
where: {
asin,
updated_at: { [Op.gte]: get_today_start() }
}
});
return !!row;
}
function unwrap_action_result(res) {
// 插件返回通常是 { ..., result: { stage, items... } }
if (res && typeof res === 'object' && res.result && typeof res.result === 'object') {
return res.result;
}
return res;
}
function normalize_sort_by(sort_by) {
if (sort_by === undefined || sort_by === null || sort_by === '') {
return null;
}
const s = String(sort_by).trim();
if (!s) {
return null;
}
const allow = new Set(['featured', 'price_asc', 'price_desc', 'review', 'newest', 'bestseller']);
if (!allow.has(s)) {
throw new Error(`sort_by 不支持: ${s}`);
}
return s;
}
async function persist_detail(detail_res_raw) {
const detail_res = unwrap_action_result(detail_res_raw);
if (!detail_res || detail_res.stage !== 'detail') {
return;
}
const asin = detail_res.asin || pick_asin_from_url(detail_res.url);
if (!asin) {
return;
}
const sku_is_object = detail_res && detail_res.sku && typeof detail_res.sku === 'object' && !Array.isArray(detail_res.sku);
await amazon_product.upsert({
asin,
url: detail_res.url || '',
title: detail_res.title || null,
price: detail_res.price || null,
sku: sku_is_object ? null : (detail_res.sku || null),
sku_json: sku_is_object ? detail_res.sku : null,
brand_line: detail_res.brand_line || null,
brand_store_url: detail_res.brand_store_url || null,
ac_badge: detail_res.ac_badge || null,
bestseller_hint: detail_res.bestseller_hint || null,
delivery_hint: detail_res.delivery_hint || null,
social_proof: detail_res.social_proof || null,
sustainability_hint: detail_res.sustainability_hint || null,
rating_stars: detail_res.rating_stars || null,
review_count_text: detail_res.review_count_text || null,
main_image: detail_res.main_image || null,
images_json: Array.isArray(detail_res.images) ? detail_res.images : null,
bullets_json: Array.isArray(detail_res.bullets) ? detail_res.bullets : null,
product_info_json: detail_res.product_info && typeof detail_res.product_info === 'object' ? detail_res.product_info : null,
detail_extra_lines_json: Array.isArray(detail_res.detail_extra_lines) ? detail_res.detail_extra_lines : null
});
}
async function persist_list(list_res_raw) {
const list_res = unwrap_action_result(list_res_raw);
if (!list_res || list_res.stage !== 'list') {
return;
}
const batch_key = build_batch_key('list');
const items = Array.isArray(list_res.items) ? list_res.items : [];
for (const it of items) {
const asin = it.asin || pick_asin_from_url(it.url);
if (!asin || !it.url) continue;
await amazon_search_item.create({
asin,
url: it.url,
title: it.title || null,
price: it.price || null,
rating: typeof it.rating === 'number' ? it.rating : null,
rating_text: it.rating_text || null,
review_count: typeof it.review_count === 'number' ? it.review_count : null,
review_count_text: it.review_count_text || null,
rank_index: typeof it.index === 'number' ? it.index : null,
batch_key,
batch_total: typeof list_res.total === 'number' ? list_res.total : null,
batch_limit: typeof list_res.limit === 'number' ? list_res.limit : null
});
}
}
async function persist_reviews(reviews_res_raw) {
const reviews_res = unwrap_action_result(reviews_res_raw);
if (!reviews_res || reviews_res.stage !== 'reviews') {
return;
}
const batch_key = build_batch_key('reviews');
const asin = pick_asin_from_url(reviews_res.url);
const items = Array.isArray(reviews_res.items) ? reviews_res.items : [];
for (const it of items) {
const review_id = it.review_id;
if (!review_id) continue;
const asin_value = asin || pick_asin_from_url(it.url) || pick_asin_from_url(reviews_res.url);
await amazon_review.upsert({
asin: asin_value || null,
url: reviews_res.url || '',
review_id,
author: it.author || null,
title: it.title || null,
body: it.body || null,
rating_text: it.rating_text || null,
review_date: it.date || null,
review_index: typeof it.index === 'number' ? it.index : null,
batch_key,
batch_total: typeof reviews_res.total === 'number' ? reviews_res.total : null,
batch_limit: typeof reviews_res.limit === 'number' ? reviews_res.limit : null
});
}
}
function must_string(v, name) {
if (typeof v !== 'string' || !v.trim()) {
throw new Error(`flow 参数 ${name} 必须是字符串`);
}
return v.trim();
}
function get_int(v, default_value) {
const n = Number(v);
if (Number.isNaN(n)) return default_value;
return n;
}
export async function run_amazon_search_detail_reviews_flow(flow_payload) {
const category_keyword = must_string(flow_payload.category_keyword, 'category_keyword');
const sort_by = normalize_sort_by(flow_payload.sort_by);
const limit = get_int(flow_payload.limit, 100);
const reviews_limit = get_int(flow_payload.reviews_limit, 50);
const gap_ms = get_int(flow_payload.gap_ms, 0);
await sleep_ms(1000);
const list_payload = { category_keyword, limit };
if (sort_by) {
list_payload.sort_by = sort_by;
}
const list_res = await execute_action_and_record({
action_name: 'amazon_search_list',
action_payload: list_payload,
source: 'cron'
});
await persist_list(list_res);
const list_result = unwrap_action_result(list_res);
if (!list_result || list_result.stage !== 'list') {
throw new Error('amazon_search_list 返回非 list stage');
}
const items = Array.isArray(list_result.items) ? list_result.items : [];
const urls = items
.map((it) => (it && it.url ? String(it.url) : ''))
.filter((u) => u);
const picked_urls = urls;
for (const url of picked_urls) {
if (gap_ms > 0) {
await sleep_ms(gap_ms);
}
const asin = pick_asin_from_url(url);
const skip_detail = asin ? await has_detail_fetched_today(asin) : false;
/**
* 合并 action同一详情页 tab 内一次完成 detail + reviews减少两次打开页面/两次桥接调用)
* - 当 skip_detail=true插件只返回 reviewsdetail 不执行/不返回)
* - 返回结构:{ result: { detail?: {...}, reviews: {...} } }
*/
const res = await execute_action_and_record({
action_name: 'amazon_product_detail_reviews',
action_payload: { product_url: url, limit: reviews_limit, skip_detail },
source: 'cron'
});
const r = unwrap_action_result(res);
const detail_part = r && r.detail ? r.detail : null;
const reviews_part = r && r.reviews ? r.reviews : null;
if (detail_part) {
await persist_detail(detail_part);
}
if (reviews_part) {
await persist_reviews(reviews_part);
}
}
const summary = {
stage: 'flow',
flow_name: 'amazon_search_detail_reviews',
category_keyword,
sort_by: sort_by || 'featured',
limit,
reviews_limit,
total_urls: urls.length,
picked: picked_urls.length
};
await close_browser();
return summary;
}

View File

@@ -0,0 +1,13 @@
import { run_amazon_search_detail_reviews_flow } from './amazon/amazon_search_detail_reviews_flow.js';
const flow_map = {
amazon_search_detail_reviews: run_amazon_search_detail_reviews_flow
};
export function get_flow_runner(flow_name) {
const fn = flow_map[flow_name];
if (!fn) {
throw new Error(`未知 flow_name: ${flow_name}`);
}
return fn;
}

View File

@@ -0,0 +1,3 @@
export async function sleep_ms(ms) {
await new Promise((resolve) => setTimeout(resolve, ms));
}

View File

@@ -0,0 +1,212 @@
import fs from 'node:fs';
import path from 'node:path';
import puppeteer from 'puppeteer';
import { get_app_config } from '../../config/app_config.js';
import { apply_page_stealth_defaults, get_stealth_puppeteer } from './puppeteer_stealth.js';
let browser_singleton = null;
export async function close_browser() {
if (!browser_singleton) {
return;
}
try {
await browser_singleton.close();
} catch (err) {
// ignore
}
browser_singleton = null;
}
function get_action_timeout_ms() {
const cfg = get_app_config();
return cfg.crawler.action_timeout_ms;
}
function get_crx_src_path() {
const cfg = get_app_config();
return cfg.crawler.crx_src_path;
}
function get_extension_id_from_targets(targets) {
for (const target of targets) {
const url = target.url();
if (!url) continue;
if (url.startsWith('chrome-extension://')) {
const match = url.match(/^chrome-extension:\/\/([^/]+)\//);
if (match && match[1]) return match[1];
}
}
return null;
}
async function wait_for_extension_id(browser, timeout_ms) {
const existing = get_extension_id_from_targets(browser.targets());
if (existing) {
return existing;
}
const target = await browser
.waitForTarget((t) => {
const url = t.url();
return typeof url === 'string' && url.startsWith('chrome-extension://');
}, { timeout: timeout_ms })
.catch(() => null);
if (!target) {
return null;
}
return get_extension_id_from_targets([target]);
}
function get_chrome_executable_path() {
const cfg = get_app_config();
return path.resolve(cfg.crawler.chrome_executable_path);
}
export async function get_or_create_browser() {
if (browser_singleton) {
return browser_singleton;
}
const chrome_executable_path = get_chrome_executable_path();
if (!fs.existsSync(chrome_executable_path)) {
throw new Error(`Chrome 不存在: ${chrome_executable_path}`);
}
const raw_extension_path = path.resolve(get_crx_src_path());
const manifest_path = path.resolve(raw_extension_path, 'manifest.json');
if (!fs.existsSync(manifest_path)) {
throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`);
}
const cfg = get_app_config();
const extension_path = raw_extension_path.replace(/\\/g, '/');
const headless = cfg.crawler.puppeteer_headless;
const cfg2 = get_app_config();
const pptr = cfg2.crawler.enable_stealth ? get_stealth_puppeteer(puppeteer) : puppeteer;
browser_singleton = await pptr.launch({
executablePath: chrome_executable_path,
headless,
args: [
'--enable-extensions',
`--disable-extensions-except=${extension_path}`,
`--load-extension=${extension_path}`,
'--no-default-browser-check',
'--disable-popup-blocking',
'--disable-dev-shm-usage',
'--disable-features=ExtensionManifestV2Disabled,ExtensionManifestV2Unsupported',
'--enable-features=AllowLegacyMV2Extensions'
]
});
return browser_singleton;
}
export async function invoke_extension_action(action_name, action_payload, options) {
const cfg = get_app_config();
const browser = await get_or_create_browser();
const started_at = Date.now();
const log_enabled = cfg.crawler.log_invoke_action;
if (log_enabled) {
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] [invoke_extension_action] start`, {
action_name,
has_payload: !!action_payload,
keys: action_payload && typeof action_payload === 'object' ? Object.keys(action_payload).slice(0, 20) : []
});
}
let page = null;
try {
page = await browser.newPage();
if (cfg.crawler.enable_stealth) {
await apply_page_stealth_defaults(page);
}
await page.goto('about:blank');
// 尝试先打开 chrome://extensions 触发扩展初始化(某些环境下扩展 target 不会立刻出现)
try {
await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' });
} catch (err) {
// ignore
}
const extension_id = await wait_for_extension_id(browser, 15000);
if (!extension_id) {
throw new Error(
'未找到扩展 extension_idChrome 未加载扩展常见原因MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless'
);
}
const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`;
await page.goto(bridge_url, { waitUntil: 'domcontentloaded' });
const timeout_ms = get_action_timeout_ms();
const action_res = await page.evaluate(
async (action, payload, timeout) => {
function with_timeout(promise, timeout_ms_inner) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner);
promise
.then((v) => {
clearTimeout(timer);
resolve(v);
})
.catch((e) => {
clearTimeout(timer);
reject(e);
});
});
}
if (!window.server_bridge_invoke) {
throw new Error('bridge 未注入 window.server_bridge_invoke');
}
return await with_timeout(window.server_bridge_invoke(action, payload), timeout);
},
action_name,
action_payload || {},
timeout_ms
);
if (log_enabled) {
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] [invoke_extension_action] ok`, {
action_name,
cost_ms: Date.now() - started_at
});
}
return action_res;
} catch (err) {
if (log_enabled) {
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] [invoke_extension_action] fail`, {
action_name,
cost_ms: Date.now() - started_at,
error: (err && err.message) || String(err)
});
}
throw err;
} finally {
if (page) {
try {
await page.close();
} catch (err) {
// ignore
}
}
const keep_browser_open = options && options.keep_browser_open === true;
if (cfg.crawler.auto_close_browser && !keep_browser_open) {
await close_browser();
}
}
}

View File

@@ -0,0 +1,27 @@
import puppeteer_extra from 'puppeteer-extra';
import stealth_plugin from 'puppeteer-extra-plugin-stealth';
// 全局只注册一次插件
let inited = false;
export function get_stealth_puppeteer(puppeteer_core) {
if (!inited) {
puppeteer_extra.use(stealth_plugin());
inited = true;
}
// 复用 puppeteer 的 Chromium/Chrome 绑定(保持你现有的 executablePath 等能力)
puppeteer_extra.puppeteer = puppeteer_core;
return puppeteer_extra;
}
export async function apply_page_stealth_defaults(page) {
// 这些属于通用的轻量“指纹一致性”设置,不会影响你现有业务
await page.setViewport({ width: 1366, height: 768, deviceScaleFactor: 1 });
await page.setUserAgent(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
);
await page.setExtraHTTPHeaders({
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
});
}

View File

@@ -1,148 +0,0 @@
import fs from 'node:fs';
import path from 'node:path';
import puppeteer from 'puppeteer';
import { get_app_config } from '../config/app_config.js';
let browser_singleton = null;
function get_action_timeout_ms() {
const cfg = get_app_config();
return cfg.crawler.action_timeout_ms;
}
function get_crx_src_path() {
const cfg = get_app_config();
return cfg.crawler.crx_src_path;
}
function get_extension_id_from_targets(targets) {
for (const target of targets) {
const url = target.url();
if (!url) continue;
if (url.startsWith('chrome-extension://')) {
const match = url.match(/^chrome-extension:\/\/([^/]+)\//);
if (match && match[1]) return match[1];
}
}
return null;
}
async function wait_for_extension_id(browser, timeout_ms) {
const existing = get_extension_id_from_targets(browser.targets());
if (existing) {
return existing;
}
const target = await browser
.waitForTarget((t) => {
const url = t.url();
return typeof url === 'string' && url.startsWith('chrome-extension://');
}, { timeout: timeout_ms })
.catch(() => null);
if (!target) {
return null;
}
return get_extension_id_from_targets([target]);
}
function get_chrome_executable_path() {
const cfg = get_app_config();
return path.resolve(cfg.crawler.chrome_executable_path);
}
export async function get_or_create_browser() {
if (browser_singleton) {
return browser_singleton;
}
const chrome_executable_path = get_chrome_executable_path();
if (!fs.existsSync(chrome_executable_path)) {
throw new Error(`Chrome 不存在: ${chrome_executable_path}`);
}
const raw_extension_path = path.resolve(get_crx_src_path());
const manifest_path = path.resolve(raw_extension_path, 'manifest.json');
if (!fs.existsSync(manifest_path)) {
throw new Error(`扩展 manifest.json 不存在: ${manifest_path}`);
}
const cfg = get_app_config();
const extension_path = raw_extension_path.replace(/\\/g, '/');
const headless = cfg.crawler.puppeteer_headless;
const user_data_dir = path.resolve(process.cwd(), 'puppeteer_profile');
browser_singleton = await puppeteer.launch({
executablePath: chrome_executable_path,
headless,
args: [
`--user-data-dir=${user_data_dir}`,
'--enable-extensions',
`--disable-extensions-except=${extension_path}`,
`--load-extension=${extension_path}`,
'--no-default-browser-check',
'--disable-popup-blocking',
'--disable-dev-shm-usage'
]
});
return browser_singleton;
}
export async function invoke_extension_action(action_name, action_payload) {
const browser = await get_or_create_browser();
const page = await browser.newPage();
await page.goto('about:blank');
// 尝试先打开 chrome://extensions 触发扩展初始化(某些环境下扩展 target 不会立刻出现)
try {
await page.goto('chrome://extensions/', { waitUntil: 'domcontentloaded' });
} catch (err) {
// ignore
}
const extension_id = await wait_for_extension_id(browser, 15000);
if (!extension_id) {
await page.close();
throw new Error(
'未找到扩展 extension_idChrome 未加载扩展常见原因MV2 被禁用/企业策略未生效/CRX_SRC_PATH 不正确/使用了 headless'
);
}
const bridge_url = `chrome-extension://${extension_id}/bridge/bridge.html`;
await page.goto(bridge_url, { waitUntil: 'domcontentloaded' });
const timeout_ms = get_action_timeout_ms();
const action_res = await page.evaluate(
async (action, payload, timeout) => {
function with_timeout(promise, timeout_ms_inner) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('action_timeout')), timeout_ms_inner);
promise
.then((v) => {
clearTimeout(timer);
resolve(v);
})
.catch((e) => {
clearTimeout(timer);
reject(e);
});
});
}
if (!window.server_bridge_invoke) {
throw new Error('bridge 未注入 window.server_bridge_invoke');
}
return await with_timeout(window.server_bridge_invoke(action, payload), timeout);
},
action_name,
action_payload || {},
timeout_ms
);
await page.close();
return action_res;
}

View File

@@ -1,24 +1,82 @@
import cron from 'node-cron';
import { cron_task_list } from '../config/cron_tasks.js';
import { execute_action_and_record } from './task_executor.js';
import { get_flow_runner } from './flows/flow_registry.js';
const cron_jobs = [];
const running_task_name_set = new Set();
/**
* 启动参数开关(用于本地调试/冷启动后立即跑一次 cron
* - 通过 VSCode/Cursor 的 launch.json 传入:--run_cron_now
* - 目的:避免等待 cron 表达式下一次触发(尤其是小时级任务)
*/
function has_argv_flag(flag_name) {
const name = String(flag_name || '').trim();
if (!name) return false;
return process.argv.includes(name);
}
function should_run_cron_now() {
return has_argv_flag('--run_cron_now');
}
async function run_cron_task(task) {
if (!task || !task.type) {
throw new Error('cron_task 缺少 type');
}
// 当前项目 cron 只允许跑 flow任务入口集中便于统一治理
if (task.type === 'flow') {
const run_flow = get_flow_runner(task.flow_name);
await run_flow(task.flow_payload || {});
return;
}
throw new Error(`cron_task type 不支持: ${task.type}`);
}
/**
* 统一的“防重复运行 + 执行 + 错误兜底”入口
* - 防止同一任务执行时间过长时,被下一次 cron 触发叠加执行
* - run_now 与定时触发复用同一套 guard保证行为一致
*/
async function run_cron_task_with_guard(task_name, task) {
if (running_task_name_set.has(task_name)) {
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] [cron] skip (already running)`, { name: task_name });
return;
}
running_task_name_set.add(task_name);
try {
await run_cron_task(task);
} catch (error) {
// eslint-disable-next-line no-console
console.warn(`[${new Date().toLocaleString()}] [cron] error`, { task_name, error });
} finally {
running_task_name_set.delete(task_name);
}
}
export async function start_all_cron_tasks() {
const run_now = should_run_cron_now();
for (const task of cron_task_list) {
// const job = cron.schedule(task.cron_expression, async () => {
try {
await execute_action_and_record({
action_name: task.action_name,
action_payload: task.action_payload || {},
source: 'cron'
});
} catch (err) {
// 失败会在 crawl_run_record 落库
}
// });
const task_name = task && task.name ? String(task.name) : 'cron_task';
// cron_jobs.push(job);
// 先注册 cron无论是否 run_now都需要后续按表达式持续执行
const job = cron.schedule(task.cron_expression, async () => {
await run_cron_task_with_guard(task_name, task);
});
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] job`, { task_name });
cron_jobs.push(job);
if (run_now) {
// 启动时额外立刻跑一次(仍走 guard避免与 cron 触发撞车)
// eslint-disable-next-line no-console
console.log(`[${new Date().toLocaleString()}] [cron] run_now`, { task_name });
await run_cron_task_with_guard(task_name, task);
}
}
}
@@ -27,4 +85,5 @@ export function stop_all_cron_tasks() {
job.stop();
}
cron_jobs.length = 0;
running_task_name_set.clear();
}

View File

@@ -1,10 +1,9 @@
import { crawl_run_record } from '../models/index.js';
import { safe_json_stringify } from './json_utils.js';
import { invoke_extension_action } from './puppeteer_runner.js';
import { persist_amazon_result } from './amazon_persist.js';
import { invoke_extension_action } from './puppeteer/puppeteer_runner.js';
export async function execute_action_and_record(params) {
const { action_name, action_payload, source } = params;
const { action_name, action_payload, source, keep_browser_open } = params;
const request_payload = safe_json_stringify(action_payload || {});
@@ -13,18 +12,14 @@ export async function execute_action_and_record(params) {
let error_message = null;
try {
const result_obj = await invoke_extension_action(action_name, action_payload || {}, {
keep_browser_open: keep_browser_open === true
});
console.log( 'invoke_extension_action-start', action_name, action_payload );
const res_invoke = await invoke_extension_action(action_name, action_payload || {});
console.log( 'invoke_extension_action-end', action_name, result );
ok = true;
result_payload = safe_json_stringify(res_invoke);
result_payload = safe_json_stringify(result_obj);
// 按 stage 自动入库(不影响原始 run_record 记录)
await persist_amazon_result(res_invoke.result);
return res_invoke;
return result_obj.result;
} catch (err) {
ok = false;
error_message = (err && err.message) || String(err);