diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c9a819637e83..c970e488d7a4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -35,6 +35,7 @@ updates: - '@oxlint/*' - 'oxfmt' - 'oxlint' + - 'oxlint-plugin-eslint' - 'oxlint-tsgolint' proxy-agent: patterns: @@ -45,6 +46,15 @@ updates: patterns: - '@typescript-eslint/*' + - package-ecosystem: 'nix' + directory: '/' + schedule: + interval: daily + time: '08:00' + open-pull-requests-limit: 100 + labels: + - dependencies + - package-ecosystem: 'github-actions' directory: '/' schedule: diff --git a/.github/workflows/comment-on-issue.yml b/.github/workflows/comment-on-issue.yml index 78371364a547..255336571baf 100644 --- a/.github/workflows/comment-on-issue.yml +++ b/.github/workflows/comment-on-issue.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies (pnpm) # import remark-parse and unified run: pnpm i - name: Generate feedback - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 0f2789c862fd..c019b02ffb9e 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -74,13 +74,13 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the Container registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -207,13 +207,13 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the Container registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/docker-test-cont.yml b/.github/workflows/docker-test-cont.yml index 7b4418845554..84b324214961 100644 --- a/.github/workflows/docker-test-cont.yml +++ b/.github/workflows/docker-test-cont.yml @@ -58,7 +58,7 @@ jobs: - name: Fetch affected routes id: fetch-route - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: PULL_REQUEST: ${{ steps.pr-data.outputs.data }} with: @@ -74,7 +74,7 @@ jobs: - name: Fetch Docker image if: (env.TEST_CONTINUE) - uses: dawidd6/action-download-artifact@8a338493df3d275e4a7a63bcff3b8fe97e51a927 # v19 + uses: dawidd6/action-download-artifact@8305c0f1062bb0d184d09ef4493ecb9288447732 # v20 with: workflow: ${{ github.event.workflow_run.workflow_id }} run_id: ${{ github.event.workflow_run.id }} @@ -127,7 +127,7 @@ jobs: if: (env.TEST_CONTINUE) id: generate-feedback timeout-minutes: 10 - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: TEST_BASEURL: http://localhost:1200 TEST_ROUTES: ${{ steps.fetch-route.outputs.result }} diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index d98c74af6f49..6fed306bcfb0 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -81,7 +81,7 @@ jobs: - name: Fetch affected routes id: fetch-route - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: EVENT: ${{ toJson(github.event) }} with: @@ -109,7 +109,7 @@ jobs: - name: Generate feedback if: env.TEST_CONTINUE - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: TEST_BASEURL: http://localhost:1200 TEST_ROUTES: ${{ steps.fetch-route.outputs.result }} diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 1bf60cacc649..39db57eb3edd 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -134,6 +134,20 @@ jobs: - Otherwise create a new PR comment. - If there are no findings and marker comment exists, edit marker comment to a short pass status. + gh command reference (PR_NUMBER and GITHUB_REPOSITORY are available as env vars): + - Create a new PR comment: + gh pr comment \"\$PR_NUMBER\" --repo \"\$GITHUB_REPOSITORY\" --body \" + ## Auto Review + ...\" + - List existing marker comments to find the one to update: + gh pr view \"\$PR_NUMBER\" --repo \"\$GITHUB_REPOSITORY\" --json comments \\ + --jq '.comments[] | select(.body | startswith(\"\")) | {id: .id, body: .body}' + - Update an existing marker comment (use the numeric id from the URL, not the node id): + gh api \"repos/\$GITHUB_REPOSITORY/issues/comments/\" -X PATCH -f body=\" + ## Auto Review + ...\" + - Prefer --body-file - with a heredoc when the body contains quotes or many lines. + Suggested comment format: ## Auto Review diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a7dd45c31b1..567b235ffdbb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,7 +145,7 @@ jobs: pull-requests: write contents: write steps: - - uses: fastify/github-action-merge-dependabot@1b2ed42db8f9d81a46bac83adedfc03eb5149dff # v3.11.2 + - uses: fastify/github-action-merge-dependabot@30c3f8f14a4f7b315ba38dbc1b793d27128fef82 # v3.12.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} target: patch diff --git a/.github/workflows/update-nix-hash.yml b/.github/workflows/update-nix-hash.yml index 8207658baed2..fb9ba223a0b6 100644 --- a/.github/workflows/update-nix-hash.yml +++ b/.github/workflows/update-nix-hash.yml @@ -5,7 +5,7 @@ on: branches: - master paths: - - 'pnpm-lock.yaml' + - 'flake.lock' permissions: contents: write @@ -20,9 +20,11 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Nix - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 + uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 with: nix_path: nixpkgs=channel:nixos-unstable + - name: Cache Nix store + uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13 - name: Update Nix flake hash id: update-hash @@ -47,7 +49,7 @@ jobs: fi # Update with correct hash - sed -i "s/hash = \"sha256-[^\"]*\";/hash = \"sha256-$NEW_HASH\";/" flake.nix + sed -i "s#hash = \"sha256-[^\"]*\";#hash = \"sha256-$NEW_HASH\";#" flake.nix if [ "$CURRENT_HASH" = "$NEW_HASH" ]; then echo "Hash unchanged" diff --git a/.oxlintrc.json b/.oxlintrc.json index 10927bb882bb..505d3386edbe 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -12,11 +12,12 @@ }, "plugins": ["eslint", "typescript", "node", "unicorn", "import"], "jsPlugins": [ - "@stylistic/eslint-plugin", { "name": "import-x-js", "specifier": "eslint-plugin-import-x" }, { "name": "n", "specifier": "eslint-plugin-n" }, { "name": "unicorn-js", "specifier": "eslint-plugin-unicorn" }, + "@stylistic/eslint-plugin", "eslint-plugin-simple-import-sort", + "oxlint-plugin-eslint", "./eslint-plugins/no-then.js", "./eslint-plugins/nsfw-flag.js" ], @@ -325,27 +326,60 @@ } ], - // "no-implicit-globals": "error", // not yet implemented + "eslint-js/no-implicit-globals": "error", // use jsPlugins "no-labels": "error", "no-lonely-if": "error", "no-multi-str": "error", "no-new-func": "error", + + "eslint-js/no-restricted-syntax": [ + "error", // use jsPlugins + { + "selector": "CallExpression[callee.property.name='get'][arguments.length=0]", + "message": "Please use .toArray() instead." + }, + { + "selector": "CallExpression[callee.property.name='toArray'] MemberExpression[object.callee.property.name='map']", + "message": "Please use .toArray() before .map()." + }, + { + "selector": "CallExpression[callee.property.name=\"catch\"] > ArrowFunctionExpression[params.length=0][body.value=null]", + "message": "Usage of .catch(() => null) is not allowed. Please handle the error appropriately." + }, + { + "selector": "CallExpression[callee.property.name=\"catch\"] > ArrowFunctionExpression[params.length=0][body.type=\"Identifier\"][body.name=\"undefined\"]", + "message": "Usage of .catch(() => undefined) is not allowed. Please handle the error appropriately." + }, + { + "selector": "CallExpression[callee.property.name=\"catch\"] > ArrowFunctionExpression[params.length=0] > ArrayExpression[elements.length=0]", + "message": "Usage of .catch(() => []) is not allowed. Please handle the error appropriately." + }, + { + "selector": "CallExpression[callee.property.name=\"catch\"] > ArrowFunctionExpression[params.length=0] > BlockStatement[body.length=0]", + "message": "Usage of .catch(() => {}) is not allowed. Please handle the error appropriately." + }, + { + "selector": "CallExpression[callee.name=\"load\"] AwaitExpression > CallExpression", + "message": "Do not use await in call expressions. Extract the result into a variable first." + } + ], + "no-unneeded-ternary": "error", "no-useless-computed-key": "error", "no-useless-concat": "warn", "no-useless-rename": "error", "no-var": "error", - // "object-shorthand": "error", // not yet implemented - // "prefer-arrow-callback'": "error", // not yet implemented + "eslint-js/object-shorthand": "error", // use jsPlugins + "eslint-js/prefer-arrow-callback": "error", // use jsPlugins "prefer-const": "error", "prefer-object-has-own": "error", + "eslint-js/prefer-regex-literals": [ + "error", // use jsPlugins + { + "disallowRedundantWrapping": true + } + ], "require-await": "error", - // "prefer-regex-literals": [ // not yet implemented - // "error", - // { - // "disallowRedundantWrapping": true - // } - // ], // #endregion // #region --- TypeScript --- @@ -354,6 +388,7 @@ "@typescript-eslint/ban-ts-comment": "off", "@typescript-eslint/consistent-indexed-object-style": "off", // stylistic "@typescript-eslint/consistent-type-definitions": "off", // stylistic + "@typescript-eslint/dot-notation": "error", // type-aware "@typescript-eslint/no-empty-function": "off", // stylistic && tests "@typescript-eslint/no-explicit-any": "off", @@ -397,6 +432,7 @@ "unicorn/no-null": "off", "unicorn/no-object-as-default-parameter": "warn", "unicorn/no-process-exit": "off", + "unicorn/no-useless-iterator-to-array": "off", "unicorn/no-useless-switch-case": "off", "unicorn/no-useless-undefined": ["error", { "checkArguments": false }], diff --git a/eslint.config.mjs b/eslint.config.mjs index 76f7be82c274..e1637cb97180 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,41 +1,26 @@ import js from '@eslint/js'; -import stylistic from '@stylistic/eslint-plugin'; import typescriptEslint from '@typescript-eslint/eslint-plugin'; import tsParser from '@typescript-eslint/parser'; // import { importX } from 'eslint-plugin-import-x'; -import n from 'eslint-plugin-n'; // import simpleImportSort from 'eslint-plugin-simple-import-sort'; import eslintPluginYml from 'eslint-plugin-yml'; import { defineConfig } from 'eslint/config'; import globals from 'globals'; -// import github from './eslint-plugins/no-then.js'; -// import nsfwFlagPlugin from './eslint-plugins/nsfw-flag.js'; - const SOURCE_FILES_GLOB = '**/*.?([cm])[jt]s?(x)'; export default defineConfig([ - // { - // plugins: { - // '@rsshub/nsfw-flag': nsfwFlagPlugin, - // }, - // rules: { - // '@rsshub/nsfw-flag/add-nsfw-flag': 'error', - // }, - // }, { ignores: ['**/coverage', '**/.vscode', '**/docker-compose.yml', '!.github', 'assets/build', 'lib/routes-deprecated', 'lib/router.js', 'dist', 'dist-lib', 'dist-worker'], }, { files: [SOURCE_FILES_GLOB], plugins: { - '@stylistic': stylistic, '@typescript-eslint': typescriptEslint, // github, js, - n, }, - // extends: [js.configs.recommended, typescriptEslint.configs['flat/recommended'], typescriptEslint.configs['flat/stylistic'], n.configs['flat/recommended-script']], + // extends: [typescriptEslint.configs['flat/recommended'], typescriptEslint.configs['flat/stylistic'], n.configs['flat/recommended-script']], languageOptions: { globals: { @@ -53,101 +38,6 @@ export default defineConfig([ }, rules: { - // #region possible problems - /* - 'array-callback-return': ['error', { allowImplicit: true }], - - 'no-await-in-loop': 'error', - 'no-control-regex': 'off', - 'no-prototype-builtins': 'off', - */ - // #endregion - - // #region suggestions - /* - 'arrow-body-style': 'error', - 'block-scoped-var': 'error', - curly: 'error', - 'dot-notation': 'error', - eqeqeq: 'error', - - 'default-case': ['warn', { commentPattern: '^no default$' }], - - 'default-case-last': 'error', - 'no-console': 'error', - 'no-eval': 'error', - 'no-extend-native': 'error', - 'no-extra-label': 'error', - - 'no-implicit-coercion': [ - 'error', - { - boolean: false, - number: false, - string: false, - disallowTemplateShorthand: true, - }, - ], - - 'no-implicit-globals': 'error', - 'no-labels': 'error', - 'no-lonely-if': 'error', - 'no-multi-str': 'error', - 'no-new-func': 'error', - */ - 'no-restricted-syntax': [ - 'error', - { - selector: "CallExpression[callee.property.name='get'][arguments.length=0]", - message: 'Please use .toArray() instead.', - }, - { - selector: "CallExpression[callee.property.name='toArray'] MemberExpression[object.callee.property.name='map']", - message: 'Please use .toArray() before .map().', - }, - { - selector: 'CallExpression[callee.property.name="catch"] > ArrowFunctionExpression[params.length=0][body.value=null]', - message: 'Usage of .catch(() => null) is not allowed. Please handle the error appropriately.', - }, - { - selector: 'CallExpression[callee.property.name="catch"] > ArrowFunctionExpression[params.length=0][body.type="Identifier"][body.name="undefined"]', - message: 'Usage of .catch(() => undefined) is not allowed. Please handle the error appropriately.', - }, - { - selector: 'CallExpression[callee.property.name="catch"] > ArrowFunctionExpression[params.length=0] > ArrayExpression[elements.length=0]', - message: 'Usage of .catch(() => []) is not allowed. Please handle the error appropriately.', - }, - { - selector: 'CallExpression[callee.property.name="catch"] > ArrowFunctionExpression[params.length=0] > BlockStatement[body.length=0]', - message: 'Usage of .catch(() => {}) is not allowed. Please handle the error appropriately.', - }, - { - selector: 'CallExpression[callee.name="load"] AwaitExpression > CallExpression', - message: 'Do not use await in call expressions. Extract the result into a variable first.', - }, - ], - /* - 'no-unneeded-ternary': 'error', - 'no-useless-computed-key': 'error', - 'no-useless-concat': 'warn', - 'no-useless-rename': 'error', - 'no-var': 'error', - 'object-shorthand': 'error', - 'prefer-arrow-callback': 'error', - 'prefer-const': 'error', - 'prefer-object-has-own': 'error', - - 'prefer-regex-literals': [ - 'error', - { - disallowRedundantWrapping: true, - }, - ], - - 'require-await': 'error', - */ - // #endregion - // #region typescript /* '@typescript-eslint/array-type': ['error', { default: 'array-simple' }], @@ -163,55 +53,6 @@ export default defineConfig([ '@typescript-eslint/no-unused-vars': ['error', { args: 'after-used', argsIgnorePattern: '^_' }], */ // #endregion - - // #region stylistic - /* - '@stylistic/arrow-parens': 'error', - '@stylistic/arrow-spacing': 'error', - '@stylistic/comma-spacing': 'error', - '@stylistic/comma-style': 'error', - '@stylistic/function-call-spacing': 'error', - '@stylistic/keyword-spacing': 'off', - '@stylistic/linebreak-style': 'error', - - '@stylistic/lines-around-comment': ['error', { beforeBlockComment: false }], - - '@stylistic/no-multiple-empty-lines': 'error', - '@stylistic/no-trailing-spaces': 'error', - '@stylistic/rest-spread-spacing': 'error', - '@stylistic/semi': 'error', - '@stylistic/space-before-blocks': 'error', - '@stylistic/space-in-parens': 'error', - '@stylistic/space-infix-ops': 'error', - '@stylistic/space-unary-ops': 'error', - '@stylistic/spaced-comment': 'error', - */ - // #endregion - - // #region node specific rules - /* - 'n/no-extraneous-require': 'error', - 'n/no-deprecated-api': 'warn', - 'n/no-missing-import': 'off', - 'n/no-missing-require': 'off', - 'n/no-process-exit': 'off', - 'n/no-unpublished-import': 'off', - - 'n/no-unpublished-require': ['error', { allowModules: ['tosource'] }], - - 'n/no-unsupported-features/node-builtins': [ - 'error', - { - version: '^22.20.0 || ^24', - allowExperimental: true, - ignores: [], - }, - ], - */ - // #endregion - - // github - // 'github/no-then': 'warn', }, }, { diff --git a/flake.lock b/flake.lock index cdfc46ed3ba8..5589a4b9f522 100644 --- a/flake.lock +++ b/flake.lock @@ -19,11 +19,11 @@ ] }, "locked": { - "lastModified": 1760971495, - "narHash": "sha256-IwnNtbNVrlZIHh7h4Wz6VP0Furxg9Hh0ycighvL5cZc=", + "lastModified": 1767714506, + "narHash": "sha256-WaTs0t1CxhgxbIuvQ97OFhDTVUGd1HA+KzLZUZBhe0s=", "owner": "cachix", "repo": "cachix", - "rev": "c5bfd933d1033672f51a863c47303fc0e093c2d2", + "rev": "894c649f0daaa38bbcfb21de64be47dfa7cd0ec9", "type": "github" }, "original": { @@ -33,21 +33,141 @@ "type": "github" } }, - "devenv": { + "cachix_2": { "inputs": { - "cachix": "cachix", + "devenv": [ + "devenv", + "crate2nix" + ], + "flake-compat": [ + "devenv", + "crate2nix" + ], + "git-hooks": "git-hooks", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1767714506, + "narHash": "sha256-WaTs0t1CxhgxbIuvQ97OFhDTVUGd1HA+KzLZUZBhe0s=", + "owner": "cachix", + "repo": "cachix", + "rev": "894c649f0daaa38bbcfb21de64be47dfa7cd0ec9", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "latest", + "repo": "cachix", + "type": "github" + } + }, + "cachix_3": { + "inputs": { + "devenv": [ + "devenv", + "crate2nix", + "crate2nix_stable" + ], + "flake-compat": [ + "devenv", + "crate2nix", + "crate2nix_stable" + ], + "git-hooks": "git-hooks_2", + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1767714506, + "narHash": "sha256-WaTs0t1CxhgxbIuvQ97OFhDTVUGd1HA+KzLZUZBhe0s=", + "owner": "cachix", + "repo": "cachix", + "rev": "894c649f0daaa38bbcfb21de64be47dfa7cd0ec9", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "latest", + "repo": "cachix", + "type": "github" + } + }, + "crate2nix": { + "inputs": { + "cachix": "cachix_2", + "crate2nix_stable": "crate2nix_stable", + "devshell": "devshell_2", + "flake-compat": "flake-compat_2", + "flake-parts": "flake-parts_2", + "nix-test-runner": "nix-test-runner_2", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "pre-commit-hooks": "pre-commit-hooks_2" + }, + "locked": { + "lastModified": 1772186516, + "narHash": "sha256-8s28pzmQ6TOIUzznwFibtW1CMieMUl1rYJIxoQYor58=", + "owner": "rossng", + "repo": "crate2nix", + "rev": "ba5dd398e31ee422fbe021767eb83b0650303a6e", + "type": "github" + }, + "original": { + "owner": "rossng", + "repo": "crate2nix", + "rev": "ba5dd398e31ee422fbe021767eb83b0650303a6e", + "type": "github" + } + }, + "crate2nix_stable": { + "inputs": { + "cachix": "cachix_3", + "crate2nix_stable": [ + "devenv", + "crate2nix", + "crate2nix_stable" + ], + "devshell": "devshell", "flake-compat": "flake-compat", "flake-parts": "flake-parts", - "git-hooks": "git-hooks", + "nix-test-runner": "nix-test-runner", + "nixpkgs": "nixpkgs_3", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1769627083, + "narHash": "sha256-SUuruvw1/moNzCZosHaa60QMTL+L9huWdsCBN6XZIic=", + "owner": "nix-community", + "repo": "crate2nix", + "rev": "7c33e664668faecf7655fa53861d7a80c9e464a2", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "0.15.0", + "repo": "crate2nix", + "type": "github" + } + }, + "devenv": { + "inputs": { + "cachix": "cachix", + "crate2nix": "crate2nix", + "flake-compat": "flake-compat_3", + "flake-parts": "flake-parts_3", + "git-hooks": "git-hooks_3", "nix": "nix", - "nixpkgs": "nixpkgs" + "nixd": "nixd", + "nixpkgs": "nixpkgs_4", + "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1764368166, - "narHash": "sha256-FktN7dtYlC/sgLGBCGFXzNOvwgB7MSujp6cooJE48Ac=", + "lastModified": 1775803833, + "narHash": "sha256-mx/CJShwE4b5CyU106ls3Jiw9fK4kyPoTLmta/cV5iM=", "owner": "cachix", "repo": "devenv", - "rev": "47a243b97499bfe5d5783d1fc86d9fe776b2497f", + "rev": "d4410df5bf2213111d3c7598b09e49176cd772d9", "type": "github" }, "original": { @@ -56,14 +176,87 @@ "type": "github" } }, + "devshell": { + "inputs": { + "nixpkgs": [ + "devenv", + "crate2nix", + "crate2nix_stable", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1768818222, + "narHash": "sha256-460jc0+CZfyaO8+w8JNtlClB2n4ui1RbHfPTLkpwhU8=", + "owner": "numtide", + "repo": "devshell", + "rev": "255a2b1725a20d060f566e4755dbf571bbbb5f76", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "devshell_2": { + "inputs": { + "nixpkgs": [ + "devenv", + "crate2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1768818222, + "narHash": "sha256-460jc0+CZfyaO8+w8JNtlClB2n4ui1RbHfPTLkpwhU8=", + "owner": "numtide", + "repo": "devshell", + "rev": "255a2b1725a20d060f566e4755dbf571bbbb5f76", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, "flake-compat": { + "locked": { + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", + "revCount": 69, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.1.0/01948eb7-9cba-704f-bbf3-3fa956735b52/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" + } + }, + "flake-compat_2": { + "locked": { + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", + "revCount": 69, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.1.0/01948eb7-9cba-704f-bbf3-3fa956735b52/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" + } + }, + "flake-compat_3": { "flake": false, "locked": { - "lastModified": 1761588595, - "narHash": "sha256-XKUZz9zewJNUj46b4AJdiRZJAvSZ0Dqj2BNfXvFlJC4=", + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", "owner": "edolstra", "repo": "flake-compat", - "rev": "f387cd2afec9419c8ee37694406ca490c3f34ee5", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", "type": "github" }, "original": { @@ -76,15 +269,60 @@ "inputs": { "nixpkgs-lib": [ "devenv", + "crate2nix", + "crate2nix_stable", "nixpkgs" ] }, "locked": { - "lastModified": 1760948891, - "narHash": "sha256-TmWcdiUUaWk8J4lpjzu4gCGxWY6/Ok7mOK4fIFfBuU4=", + "lastModified": 1768135262, + "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "864599284fc7c0ba6357ed89ed5e2cd5040f0c04", + "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "devenv", + "crate2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1768135262, + "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_3": { + "inputs": { + "nixpkgs-lib": [ + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1772408722, + "narHash": "sha256-rHuJtdcOjK7rAHpHphUb1iCvgkU3GpfvicLMwwnfMT0=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "f20dc5d9b8027381c474144ecabc9034d6a839a3", "type": "github" }, "original": { @@ -115,20 +353,82 @@ "inputs": { "flake-compat": [ "devenv", + "crate2nix", + "cachix", "flake-compat" ], "gitignore": "gitignore", "nixpkgs": [ "devenv", + "crate2nix", + "cachix", "nixpkgs" ] }, "locked": { - "lastModified": 1760663237, - "narHash": "sha256-BflA6U4AM1bzuRMR8QqzPXqh8sWVCNDzOdsxXEguJIc=", + "lastModified": 1765404074, + "narHash": "sha256-+ZDU2d+vzWkEJiqprvV5PR26DVFN2vgddwG5SnPZcUM=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "ca5b894d3e3e151ffc1db040b6ce4dcc75d31c37", + "rev": "2d6f58930fbcd82f6f9fd59fb6d13e37684ca529", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "git-hooks_2": { + "inputs": { + "flake-compat": [ + "devenv", + "crate2nix", + "crate2nix_stable", + "cachix", + "flake-compat" + ], + "gitignore": "gitignore_2", + "nixpkgs": [ + "devenv", + "crate2nix", + "crate2nix_stable", + "cachix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1765404074, + "narHash": "sha256-+ZDU2d+vzWkEJiqprvV5PR26DVFN2vgddwG5SnPZcUM=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "2d6f58930fbcd82f6f9fd59fb6d13e37684ca529", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "git-hooks_3": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "gitignore": "gitignore_5", + "nixpkgs": [ + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1772893680, + "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9", "type": "github" }, "original": { @@ -138,6 +438,102 @@ } }, "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv", + "crate2nix", + "cachix", + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "gitignore_2": { + "inputs": { + "nixpkgs": [ + "devenv", + "crate2nix", + "crate2nix_stable", + "cachix", + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "gitignore_3": { + "inputs": { + "nixpkgs": [ + "devenv", + "crate2nix", + "crate2nix_stable", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "gitignore_4": { + "inputs": { + "nixpkgs": [ + "devenv", + "crate2nix", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "gitignore_5": { "inputs": { "nixpkgs": [ "devenv", @@ -185,27 +581,153 @@ ] }, "locked": { - "lastModified": 1761648602, - "narHash": "sha256-H97KSB/luq/aGobKRuHahOvT1r7C03BgB6D5HBZsbN8=", + "lastModified": 1775657489, + "narHash": "sha256-v1KwZrIMGpteHPwxXvbapc7o3iduhU61phPUfyrnjM8=", "owner": "cachix", "repo": "nix", - "rev": "3e5644da6830ef65f0a2f7ec22830c46285bfff6", + "rev": "5c0da4397902105a84611c6d49e9d39a618ca025", "type": "github" }, "original": { "owner": "cachix", - "ref": "devenv-2.30.6", + "ref": "devenv-2.34", "repo": "nix", "type": "github" } }, + "nix-test-runner": { + "flake": false, + "locked": { + "lastModified": 1588761593, + "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=", + "owner": "stoeffel", + "repo": "nix-test-runner", + "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2", + "type": "github" + }, + "original": { + "owner": "stoeffel", + "repo": "nix-test-runner", + "type": "github" + } + }, + "nix-test-runner_2": { + "flake": false, + "locked": { + "lastModified": 1588761593, + "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=", + "owner": "stoeffel", + "repo": "nix-test-runner", + "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2", + "type": "github" + }, + "original": { + "owner": "stoeffel", + "repo": "nix-test-runner", + "type": "github" + } + }, + "nixd": { + "inputs": { + "flake-parts": [ + "devenv", + "flake-parts" + ], + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1773634079, + "narHash": "sha256-49qb4QNMv77VOeEux+sMd0uBhPvvHgVc0r938Bulvbo=", + "owner": "nix-community", + "repo": "nixd", + "rev": "8ecf93d4d93745e05ea53534e8b94f5e9506e6bd", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixd", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1761313199, - "narHash": "sha256-wCIACXbNtXAlwvQUo1Ed++loFALPjYUA3dpcUJiXO44=", + "lastModified": 1765186076, + "narHash": "sha256-hM20uyap1a0M9d344I692r+ik4gTMyj60cQWO+hAYP8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "addf7cf5f383a3101ecfba091b98d0a1263dc9b8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-src": { + "flake": false, + "locked": { + "lastModified": 1773597492, + "narHash": "sha256-hQ284SkIeNaeyud+LS0WVLX+WL2rxcVZLFEaK0e03zg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a07d4ce6bee67d7c838a8a5796e75dff9caa21ef", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1765186076, + "narHash": "sha256-hM20uyap1a0M9d344I692r+ik4gTMyj60cQWO+hAYP8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "addf7cf5f383a3101ecfba091b98d0a1263dc9b8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1769433173, + "narHash": "sha256-Gf1dFYgD344WZ3q0LPlRoWaNdNQq8kSBDLEWulRQSEs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "13b0f9e6ac78abbbb736c635d87845c4f4bee51b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1773704619, + "narHash": "sha256-LKtmit8Sr81z8+N2vpIaN/fyiQJ8f7XJ6tMSKyDVQ9s=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "d1c30452ebecfc55185ae6d1c983c09da0c274ff", + "rev": "906534d75b0e2fe74a719559dfb1ad3563485f43", "type": "github" }, "original": { @@ -215,13 +737,13 @@ "type": "github" } }, - "nixpkgs_2": { + "nixpkgs_5": { "locked": { - "lastModified": 1764242076, - "narHash": "sha256-sKoIWfnijJ0+9e4wRvIgm/HgE27bzwQxcEmo2J/gNpI=", + "lastModified": 1775710090, + "narHash": "sha256-ar3rofg+awPB8QXDaFJhJ2jJhu+KqN/PRCXeyuXR76E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2fad6eac6077f03fe109c4d4eb171cf96791faa4", + "rev": "4c1018dae018162ec878d42fec712642d214fdfa", "type": "github" }, "original": { @@ -231,11 +753,90 @@ "type": "github" } }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "devenv", + "crate2nix", + "crate2nix_stable", + "flake-compat" + ], + "gitignore": "gitignore_3", + "nixpkgs": [ + "devenv", + "crate2nix", + "crate2nix_stable", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769069492, + "narHash": "sha256-Efs3VUPelRduf3PpfPP2ovEB4CXT7vHf8W+xc49RL/U=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "a1ef738813b15cf8ec759bdff5761b027e3e1d23", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "pre-commit-hooks_2": { + "inputs": { + "flake-compat": [ + "devenv", + "crate2nix", + "flake-compat" + ], + "gitignore": "gitignore_4", + "nixpkgs": [ + "devenv", + "crate2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769069492, + "narHash": "sha256-Efs3VUPelRduf3PpfPP2ovEB4CXT7vHf8W+xc49RL/U=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "a1ef738813b15cf8ec759bdff5761b027e3e1d23", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, "root": { "inputs": { "devenv": "devenv", "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_5" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1773630837, + "narHash": "sha256-zJhgAGnbVKeBMJOb9ctZm4BGS/Rnrz+5lfSXTVah4HQ=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "f600ea449c7b5bb596fa1cf21c871cc5b9e31316", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" } }, "systems": { @@ -252,6 +853,28 @@ "repo": "default", "type": "github" } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "devenv", + "nixd", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1772660329, + "narHash": "sha256-IjU1FxYqm+VDe5qIOxoW+pISBlGvVApRjiw/Y/ttJzY=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "3710e0e1218041bbad640352a0440114b1e10428", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 563a42c91332..b2422bf5d285 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,7 @@ deps = pkgs.fetchPnpmDeps { pname = "rsshub"; src = ./.; - hash = "sha256-QG1cIkZh+qBA5Dipt0iDLuQpEOI45wdFhuG/CTcRVU8="; + hash = "sha256-Q7plMp6xtB7tnTyRu2/ik8pY0kXt82BLNBQ3lG3dHSo="; fetcherVersion = 2; }; in diff --git a/lib/routes/8kcos/utils.ts b/lib/routes/8kcos/utils.ts index 9bb08bc5b04c..237317eaec81 100644 --- a/lib/routes/8kcos/utils.ts +++ b/lib/routes/8kcos/utils.ts @@ -19,7 +19,7 @@ export const getPosts = async (limit: number, options?: { categories?: number; t description: item.content.rendered, link: item.link, pubDate: parseDate(item.date_gmt), - author: item._embedded?.['author']?.map((a) => a.name).join(', '), + author: item._embedded?.author?.map((a) => a.name).join(', '), category: item._embedded?.['wp:term']?.flatMap((terms) => terms.map((t) => t.name)), })) satisfies DataItem[]; }; diff --git a/lib/routes/acfun/bangumi.ts b/lib/routes/acfun/bangumi.ts index b83c04319c1e..01be818d10df 100644 --- a/lib/routes/acfun/bangumi.ts +++ b/lib/routes/acfun/bangumi.ts @@ -1,14 +1,16 @@ import type { Route } from '@/types'; import { ViewType } from '@/types'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +import { renderDescription } from './templates/description'; + export const route: Route = { - path: '/bangumi/:id', + path: '/bangumi/:id/:embed?', categories: ['anime'], view: ViewType.Videos, - example: '/acfun/bangumi/5022158', - parameters: { id: '番剧 id' }, + example: '/acfun/bangumi/6000617', + parameters: { id: '番剧 id', embed: '默认为开启内嵌视频, 任意值为关闭' }, features: { requireConfig: false, requirePuppeteer: false, @@ -22,31 +24,28 @@ export const route: Route = { handler, description: `::: tip 番剧 id 不包含开头的 aa。 -例如：\`https://www.acfun.cn/bangumi/aa5022158\` 的番剧 id 是 5022158，不包括开头的 aa。 +例如：\`https://www.acfun.cn/bangumi/aa6000617\` 的番剧 id 是 6000617，不包括开头的 aa。 :::`, }; async function handler(ctx) { const id = ctx.req.param('id'); - const url = `https://www.acfun.cn/bangumi/aa${id}`; + const embed = !ctx.req.param('embed'); + const link = `https://www.acfun.cn/bangumi/aa${id}`; - const bangumiPage = await got(url, { - headers: { - Referer: 'https://www.acfun.cn', - }, - }); - const bangumiData = JSON.parse(bangumiPage.data.match(/window.bangumiData = (.*?);\n/)[1]); - const bangumiList = JSON.parse(bangumiPage.data.match(/window.bangumiList = (.*?);\n/)[1]); + const bangumiPage = await ofetch(link); + const bangumiData = JSON.parse(bangumiPage.match(/window.bangumiData = (.*?);\n/)[1]); + const bangumiList = JSON.parse(bangumiPage.match(/window.bangumiList = (.*?);\n/)[1]); return { title: bangumiData.bangumiTitle, - link: url, + link, description: bangumiData.bangumiIntro, image: bangumiData.belongResource.coverImageV, item: bangumiList.items.map((item) => ({ - title: `${item.episodeName} - ${item.title}`, - description: ``, - link: `http://www.acfun.cn/bangumi/aa${id}_36188_${item.itemId}`, + title: `${item.episodeName}${item.title ? ` - ${item.title}` : ''}`, + description: renderDescription({ embed, aid: `ac${item.itemId}`, img: item.imgInfo.thumbnailImage.cdnUrls[0].url.split('?')[0] }), + link: `https://www.acfun.cn/bangumi/aa${id}_36188_${item.itemId}`, pubDate: parseDate(item.updateTime, 'x'), })), }; diff --git a/lib/routes/acfun/templates/description.tsx b/lib/routes/acfun/templates/description.tsx new file mode 100644 index 000000000000..04d48f07c198 --- /dev/null +++ b/lib/routes/acfun/templates/description.tsx @@ -0,0 +1,21 @@ +import { renderToString } from 'hono/jsx/dom/server'; + +type DescriptionProps = { + embed: boolean; + aid: string; + img?: string; +}; + +const Description = ({ embed, aid, img }: DescriptionProps) => ( + <> + {embed ? ( + <> + +
+ + ) : null} + {img ? : null} + +); + +export const renderDescription = (props: DescriptionProps): string => renderToString(); diff --git a/lib/routes/acfun/video.ts b/lib/routes/acfun/video.ts index 1c105702a3f0..39ca0200be73 100644 --- a/lib/routes/acfun/video.ts +++ b/lib/routes/acfun/video.ts @@ -2,11 +2,13 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import { ViewType } from '@/types'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +import { renderDescription } from './templates/description'; + export const route: Route = { - path: '/user/video/:uid', + path: '/user/video/:uid/:embed?', radar: [ { source: ['www.acfun.cn/u/:id'], @@ -16,6 +18,7 @@ export const route: Route = { name: '用户投稿', parameters: { uid: '用户 UID', + embed: '默认为开启内嵌视频, 任意值为关闭', }, categories: ['anime'], example: '/acfun/user/video/6102', @@ -26,39 +29,37 @@ export const route: Route = { async function handler(ctx) { const uid = ctx.req.param('uid'); - const url = `https://www.acfun.cn/u/${uid}`; + const embed = !ctx.req.param('embed'); const host = 'https://www.acfun.cn'; - const response = await got(url, { - headers: { - Referer: host, - }, - }); - const data = response.data; + const link = `${host}/u/${uid}`; + const response = await ofetch(link); - const $ = load(data); + const $ = load(response); const title = $('title').text(); const description = $('.signature .complete').text(); const list = $('#ac-space-video-list a').toArray(); - const image = $('head style') + const image = $('head style:contains("user-photo")') .text() - .match(/.user-photo{\n\s*background:url\((.*)\) 0% 0% \/ 100% no-repeat;/)[1]; + .match(/.user-photo{\n\s*background:url\((.*)\) 0% 0% \/ 100% no-repeat;/)?.[1]; return { title, - link: url, + link, description, image, item: list.map((item) => { - item = $(item); + const $item = $(item); - const itemTitle = item.find('p.title').text(); - const itemImg = item.find('figure img').attr('src'); - const itemUrl = item.attr('href'); - const itemDate = item.find('.date').text(); + const itemTitle = $item.find('p.title').text(); + const itemImg = $item.find('figure img').attr('src'); + const itemUrl = $item.attr('href')!; + const itemDate = $item.find('.date').text(); + const wbInfo = JSON.parse(($item.data('wb-info') as string) || '{}'); + const aid = wbInfo.atmid || wbInfo.mediaId || itemUrl.match(/\/v\/(ac\d+)/)?.[1]; return { title: itemTitle, - description: ``, + description: renderDescription({ embed, aid, img: itemImg?.split('?')[0] }), link: host + itemUrl, pubDate: parseDate(itemDate, 'YYYY/MM/DD'), }; diff --git a/lib/routes/bilibili/cache.ts b/lib/routes/bilibili/cache.ts index 63c3a201f601..e4add3e2226c 100644 --- a/lib/routes/bilibili/cache.ts +++ b/lib/routes/bilibili/cache.ts @@ -38,7 +38,7 @@ const getCookie = (disableConfig = false) => { let waitForRequest = new Promise((resolve) => { resolve(''); }); - const { destory } = await getPuppeteerPage('https://space.bilibili.com/1/dynamic', { + const { destroy } = await getPuppeteerPage('https://space.bilibili.com/1/dynamic', { onBeforeLoad: (page) => { waitForRequest = new Promise((resolve) => { page.on('requestfinished', async (request) => { @@ -54,7 +54,7 @@ const getCookie = (disableConfig = false) => { }); const cookieString = await waitForRequest; logger.debug(`Got bilibili cookie: ${cookieString}`); - await destory(); + await destroy(); return cookieString; }); }; diff --git a/lib/routes/castanet/namespace.ts b/lib/routes/castanet/namespace.ts new file mode 100644 index 000000000000..02f37eae4eab --- /dev/null +++ b/lib/routes/castanet/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Castanet', + url: 'www.castanet.net', + lang: 'en', +}; diff --git a/lib/routes/castanet/news.ts b/lib/routes/castanet/news.ts new file mode 100644 index 000000000000..d9f4894a6e8f --- /dev/null +++ b/lib/routes/castanet/news.ts @@ -0,0 +1,104 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import parser from '@/utils/rss-parser'; + +const feeds = { + 'Top Headlines': 'topheadlines', + 'Recent Headlines': 'mostrecent', + Kelowna: 'page-1', + 'West-Kelowna': 'page-101', + Peachland: 'page-86', + Vernon: 'page-2', + 'Salmon-Arm': 'page-61', + Penticton: 'page-21', + 'Oliver-Osoyoos': 'page-87', + Kamloops: 'page-48', + Nelson: 'page-91', + BC: 'page-3', + Canada: 'page-4', + World: 'page-5', + Business: 'page-6', + Sports: 'page-7', + ShowBiz: 'page-8', +}; + +export const route: Route = { + path: '/:category?', + categories: ['traditional-media'], + example: '/castanet/Kelowna', + parameters: { + category: { + options: Object.keys(feeds).map((k) => ({ + value: k, + label: k, + })), + description: 'Category', + default: 'Recent Headlines', + }, + }, + radar: [ + { + source: ['www.castanet.net/news/:category/'], + target: '/:category', + }, + { + source: ['www.castanet.net/'], + target: '/', + }, + ], + name: 'News', + maintainers: ['TonyRL'], + handler, + url: 'www.castanet.net', +}; + +async function handler(ctx) { + const category = ctx.req.param('category') ?? 'Recent Headlines'; + const baseUrl = 'https://www.castanet.net'; + const feedFile = feeds[category] ?? category; + const feedUrl = `${baseUrl}/rss/${feedFile}.xml`; + + const feed = await parser.parseURL(feedUrl); + + const items = await Promise.all( + feed.items.map((item) => + cache.tryGet(item.link as string, async () => { + const response = await ofetch(item.link as string); + const $ = load(response); + + delete item.content; + delete item.contentSnippet; + delete item.isoDate; + + const content = $('.newsstory'); + content.find('.newsheadlinefull, .newsheadline, .byline, .click_gallery').remove(); + if (content.find('.gallery_img1').length) { + const a = content.find('.gallery_img1 a').toArray(); + content.find('.gallery_img1').next().remove(); + for (const ele of a) { + const $ele = $(ele); + const href = $ele.attr('href'); + const figure = `
${$ele.text().trim()}
${$ele.attr('title')?.trim() ?? ''}
`; + $ele.replaceWith(figure); + } + } + + item.description = content.html()?.trim(); + + return item; + }) + ) + ); + + return { + title: feed.title, + link: feed.link, + description: feed.description, + image: feed.image?.url, + language: feed.language, + item: items, + }; +} diff --git a/lib/routes/castbox/channel.ts b/lib/routes/castbox/channel.ts new file mode 100644 index 000000000000..335b520f8f59 --- /dev/null +++ b/lib/routes/castbox/channel.ts @@ -0,0 +1,113 @@ +import crypto from 'node:crypto'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +const PERMUTAION_MAP = [24, 13, 4, 19, 6, 0, 8, 21, 25, 7, 28, 1, 15, 31, 10, 9, 17, 18, 22, 11, 27, 23, 2, 26, 12, 5, 29, 14, 20, 30, 16, 3]; + +const getNonce = (params: Record) => { + const m = new Date().toISOString().slice(0, 10).replaceAll('-', ''); + + const sortedKeys = Object.keys(params).toSorted(); + const queryParts = sortedKeys.map((k) => `${k}=${params[k]}`); + const queryStr = queryParts.join('&'); + + const hashInput = `${queryStr}evst${m}`; + + const md5Hex = crypto.createHash('md5').update(hashInput).digest('hex'); + + const n = PERMUTAION_MAP.map((idx) => md5Hex[idx]).join(''); + + return { m, n, queryStr }; +}; + +export const route: Route = { + path: '/channel/:channel', + categories: ['multimedia'], + example: '/castbox/channel/Lemonade-Stand-id6776228', + parameters: { + channel: 'Channel', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: true, + supportScihub: false, + }, + radar: [ + { + source: ['castbox.fm/channel/:channel'], + target: '/channel/:channel', + }, + ], + name: 'Channels', + description: `Get the channel from the Castbox channel URL. For example, the URL of the channel "Lemonade Stand" is \`https://castbox.fm/channel/Lemonade-Stand-id6776228\`, where \`Lemonade-Stand-id6776228\` is the \`channel\` parameter. + + You can use the RSSHub global \`limit\` query parameter to specify the maximum number of episodes to fetch from the Castbox API (defaults to 50). For example: \`/castbox/channel/Lemonade-Stand-id6776228?limit=100\`.`, + maintainers: ['ananyatimalsina'], + handler: async (ctx) => { + const { channel } = ctx.req.param(); + const cid = channel.split('-id')[1]; + + if (!cid) { + throw new Error('Invalid channel format. Missing -id'); + } + + const channelParams = { cid, r: 1, raw: 1, web: 1 }; + const { m: cm, n: cn, queryStr: cQuery } = getNonce(channelParams); + + const channelData = await ofetch(`https://everest.castbox.fm/data/channel/v3?${cQuery}&m=${cm}&n=${cn}`); + + if (!channelData?.data) { + throw new Error('Failed to fetch channel data from Castbox'); + } + + const chData = channelData.data; + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit') as string, 10) : 50; + + const epParams = { cid, limit, r: 1, raw: 1, web: 1 }; + const { m: em, n: en, queryStr: eQuery } = getNonce(epParams); + + const epData = await ofetch(`https://everest.castbox.fm/data/episode_list/v2?${eQuery}&m=${em}&n=${en}`); + + if (!epData?.data?.episode_list) { + throw new Error('Failed to fetch episode list from Castbox'); + } + + const episodes = epData.data.episode_list; + + const items = episodes.map((ep: any) => { + let enclosure_type = 'audio/mpeg'; + if (ep.video === 1 || ep.url?.includes('.mp4')) { + enclosure_type = 'video/mp4'; + } else if (ep.url?.includes('.m4a')) { + enclosure_type = 'audio/mp4'; + } + + return { + title: ep.title, + description: ep.description, + pubDate: parseDate(ep.release_date), + link: `https://castbox.fm/episode/${encodeURIComponent(ep.title)}-id${cid}-id${ep.eid}`, + enclosure_url: ep.url, + enclosure_type, + enclosure_length: ep.size, + itunes_item_image: ep.big_cover_url || ep.cover_url, + itunes_duration: ep.duration ? Math.round(ep.duration / 1000) : undefined, + }; + }); + + return { + title: chData.title, + link: `https://castbox.fm/channel/${channel}`, + description: chData.description, + image: chData.big_cover_url || chData.small_cover_url, + language: chData.language, + itunes_author: chData.author, + item: items, + }; + }, +}; diff --git a/lib/routes/castbox/namespace.ts b/lib/routes/castbox/namespace.ts new file mode 100644 index 000000000000..d7b35eda6643 --- /dev/null +++ b/lib/routes/castbox/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Castbox', + url: 'castbox.fm', + description: `Castbox is a podcast distribution network and producer.`, +}; diff --git a/lib/routes/cjlu/yjsy/index.ts b/lib/routes/cjlu/yjsy/index.ts index 1b333b623d12..894e488bfdce 100644 --- a/lib/routes/cjlu/yjsy/index.ts +++ b/lib/routes/cjlu/yjsy/index.ts @@ -86,7 +86,7 @@ async function handler(ctx) { const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 10; const url = `${host}index/${cate}.htm`; - const { page, destory, browser } = await getPuppeteerPage(url, { + const { page, destroy, browser } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setExtraHTTPHeaders(headers); await page.setUserAgent(headers['User-Agent']); @@ -102,7 +102,7 @@ async function handler(ctx) { const cookieString = cookies.map((c) => `${c.name}=${c.value}`).join('; '); const response = await page.content(); - await destory(); + await destroy(); const $ = load(response); diff --git a/lib/routes/claude/code-changelog.ts b/lib/routes/claude/code-changelog.ts index c5697ac52bc9..7d8483231530 100644 --- a/lib/routes/claude/code-changelog.ts +++ b/lib/routes/claude/code-changelog.ts @@ -4,6 +4,7 @@ import type { Context } from 'hono'; import type { Data, DataItem, Route } from '@/types'; import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; const handler = async (ctx: Context): Promise => { const limit = Number.parseInt(ctx.req.query('limit') ?? '25', 10); @@ -14,29 +15,27 @@ const handler = async (ctx: Context): Promise => { const response = await ofetch(targetUrl); const $: CheerioAPI = load(response); - const items: DataItem[] = $('div.markdown-heading') + const items: DataItem[] = $('div.update-container') .slice(0, limit) .toArray() .map((el): DataItem => { - const $heading = $(el); - const version = $heading.find('h2.heading-element').text().trim(); + const $entry = $(el); + const version = $entry.find('[data-component-part="update-label"]').text().trim(); if (!version) { return null as unknown as DataItem; } - const descriptionParts: string[] = []; - $heading.nextUntil('div.markdown-heading').each((_, sibling) => { - descriptionParts.push($(sibling).prop('outerHTML') ?? ''); - }); - const description = descriptionParts.join(''); + const dateText = $entry.find('[data-component-part="update-description"]').text().trim(); + const description = $entry.find('[data-component-part="update-content"]').html() ?? ''; - const anchor = $heading.find('a.anchor').attr('href') ?? `#${version.replaceAll('.', '')}`; - const link = `${targetUrl}${anchor}`; + const anchor = $entry.attr('id') ?? version.replaceAll('.', '-'); + const link = `${targetUrl}#${anchor}`; return { title: version, description, link, + pubDate: dateText ? parseDate(dateText) : undefined, guid: `claude-code-${version}`, id: `claude-code-${version}`, }; diff --git a/lib/routes/dailypush/all.ts b/lib/routes/dailypush/all.ts index 8e0ed6fa3baa..d15bb923de9a 100644 --- a/lib/routes/dailypush/all.ts +++ b/lib/routes/dailypush/all.ts @@ -1,9 +1,9 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; -import ofetch from '@/utils/ofetch'; +import puppeteer from '@/utils/puppeteer'; -import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils'; +import { BASE_URL, enhanceItemsWithSummaries, fetchPageHtml, parseArticles } from './utils'; export const route: Route = { path: '/:sort?', @@ -21,7 +21,7 @@ export const route: Route = { }, features: { requireConfig: false, - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -42,17 +42,21 @@ async function handler(ctx) { const { sort = '' } = ctx.req.param(); const url = sort ? `${BASE_URL}/${sort}` : BASE_URL; - const response = await ofetch(url); - const $ = load(response); - - const list = parseArticles($, BASE_URL); - const items = await enhanceItemsWithSummaries(list); - - const pageTitle = $('title').text() || 'DailyPush - All'; - - return { - title: pageTitle, - link: url, - item: items, - }; + const browser = await puppeteer(); + try { + const html = await fetchPageHtml(browser, url, 'article'); + const $ = load(html); + const list = parseArticles($, BASE_URL); + const items = await enhanceItemsWithSummaries(browser, list); + + const pageTitle = $('title').text() || 'DailyPush - All'; + + return { + title: pageTitle, + link: url, + item: items, + }; + } finally { + await browser.close(); + } } diff --git a/lib/routes/dailypush/tags.ts b/lib/routes/dailypush/tags.ts index 1c4e8bf58387..da50a1556210 100644 --- a/lib/routes/dailypush/tags.ts +++ b/lib/routes/dailypush/tags.ts @@ -1,9 +1,9 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; -import ofetch from '@/utils/ofetch'; +import puppeteer from '@/utils/puppeteer'; -import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils'; +import { BASE_URL, enhanceItemsWithSummaries, fetchPageHtml, parseArticles } from './utils'; export const route: Route = { path: '/tag/:tag/:sort?', @@ -22,7 +22,7 @@ export const route: Route = { }, features: { requireConfig: false, - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -43,17 +43,21 @@ async function handler(ctx) { const { tag, sort = 'trending' } = ctx.req.param(); const url = `${BASE_URL}/${tag}/${sort}`; - const response = await ofetch(url); - const $ = load(response); - - const list = parseArticles($, BASE_URL); - const items = await enhanceItemsWithSummaries(list); - - const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`; - - return { - title: pageTitle, - link: url, - item: items, - }; + const browser = await puppeteer(); + try { + const html = await fetchPageHtml(browser, url, 'article'); + const $ = load(html); + const list = parseArticles($, BASE_URL); + const items = await enhanceItemsWithSummaries(browser, list); + + const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`; + + return { + title: pageTitle, + link: url, + item: items, + }; + } finally { + await browser.close(); + } } diff --git a/lib/routes/dailypush/utils.ts b/lib/routes/dailypush/utils.ts index 041eaa600ed0..2854207eae8c 100644 --- a/lib/routes/dailypush/utils.ts +++ b/lib/routes/dailypush/utils.ts @@ -1,9 +1,10 @@ import type { CheerioAPI } from 'cheerio'; import { load } from 'cheerio'; +import type { Browser, Page } from 'rebrowser-puppeteer'; import type { DataItem } from '@/types'; import cache from '@/utils/cache'; -import ofetch from '@/utils/ofetch'; +import logger from '@/utils/logger'; import { parseRelativeDate } from '@/utils/parse-date'; export const BASE_URL = 'https://www.dailypush.dev'; @@ -19,6 +20,38 @@ export interface ArticleItem { dailyPushUrl?: string; } +const allowedRequestTypes = new Set(['document']); + +async function preparePage(page: Page) { + await page.setRequestInterception(true); + page.on('request', (request) => { + if (allowedRequestTypes.has(request.resourceType())) { + request.continue(); + return; + } + + request.abort(); + }); +} + +export async function fetchPageHtml(browser: Browser, url: string, waitForSelector?: string): Promise { + const page = await browser.newPage(); + await preparePage(page); + + try { + logger.http(`Requesting ${url}`); + await page.goto(url, { waitUntil: 'domcontentloaded' }); + + if (waitForSelector) { + await page.waitForSelector(waitForSelector); + } + + return await page.content(); + } finally { + await page.close(); + } +} + /** * Try to parse text as a date. Returns the Date if parsing succeeds and is valid, undefined otherwise. */ @@ -40,14 +73,14 @@ function extractAuthor(article: ReturnType): DataItem['author'] { return undefined; } - // Get all content spans (exclude separator spans with '•') + // Get all content spans (exclude separator spans with "•") const allSpans = container.find('span'); const contentSpans: string[] = []; for (let i = 0; i < allSpans.length; i++) { const $span = allSpans.eq(i); const text = $span.text().trim(); - // Skip separator spans (contain only '•' or have separator classes) + // Skip separator spans (contain only "•" or have separator classes) if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) { contentSpans.push(text); } @@ -127,14 +160,14 @@ function extractPubDate(article: ReturnType): Date | undefined { return undefined; } - // Get all content spans (exclude separator spans with '•') + // Get all content spans (exclude separator spans with "•") const allSpans = container.find('span'); const contentSpans: string[] = []; for (let i = 0; i < allSpans.length; i++) { const $span = allSpans.eq(i); const text = $span.text().trim(); - // Skip separator spans (contain only '•' or have separator classes) + // Skip separator spans (contain only "•" or have separator classes) if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) { contentSpans.push(text); } @@ -225,23 +258,20 @@ export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] { } /** - * Enhance items with full summaries from dailypush article pages + * Enhance items with full summaries from dailypush article pages. + * Uses the provided browser; opens a new tab per URL (document requests only). Caller must close the browser. */ -export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise { +export async function enhanceItemsWithSummaries(browser: Browser, items: ArticleItem[]): Promise { const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined); const itemsWithoutUrl: DataItem[] = items.filter((item) => item.dailyPushUrl === undefined); - const enhancedItems: DataItem[] = await Promise.all( + const enhancedItems = await Promise.all( itemsWithUrl.map((item) => cache.tryGet(item.dailyPushUrl!, async () => { - // If we have a dailypush article URL, fetch it for the longer summary try { - const articleResponse = await ofetch(item.dailyPushUrl!); - const $ = load(articleResponse); - - // Find the longer summary/description on the article page + const html = await fetchPageHtml(browser, item.dailyPushUrl!, 'p.font-ibm-plex-sans.leading-relaxed'); + const $ = load(html); const summary = $('p.font-ibm-plex-sans.leading-relaxed').first(); - if (summary.length > 0 && summary.text().trim()) { item.description = summary.text().trim(); } @@ -254,6 +284,5 @@ export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise { + const response = await ofetch('https://www.fcbayern.cn/api2018/news/list', { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + page: '1', + pageSize: String(limit), + totalPage: '1', + tagId: '', + year: '-1', + month: '-1', + type: '1', + }), + }); + + const items = response.data.map((item) => ({ + title: item.title, + link: item.url, + pubDate: timezone(parseDate(item.time), 8), + image: item.pic?.split('?')[0], + })); + + return { + title: '拜仁慕尼黑俱乐部中文官方网站 - 新闻', + link: 'https://www.fcbayern.cn/news', + item: items, + } satisfies Data; +}; diff --git a/lib/routes/fcbayern/news.ts b/lib/routes/fcbayern/news.ts new file mode 100644 index 000000000000..2a3d7c8d50f6 --- /dev/null +++ b/lib/routes/fcbayern/news.ts @@ -0,0 +1,207 @@ +import type { Context } from 'hono'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import { cnNewsHander } from './news-cn'; + +const languages = { + en: 'fcbayern.com-en-gb', + es: 'fcbayern.com-es-es', + de: 'fcbayern.com-de-de', +}; + +const query = /* GraphQL */ ` + query Web_NewsSearch($channelId: String!, $query: String!, $filterTags: [String!], $filterSections: [String!], $filterTypes: [String!], $count: Int!, $offset: Int, $excludeHeroStageResultFromPageId: String) { + newsSearch( + channelId: $channelId + query: $query + filterTags: $filterTags + filterSections: $filterSections + filterTypes: $filterTypes + count: $count + offset: $offset + excludeHeroStageResultFromPageId: $excludeHeroStageResultFromPageId + ) { + total + offset + count + results { + ...Teaser + __typename + } + __typename + } + } + fragment Teaser on Teaser { + __typename + id + pageType + teaserTitle + teaserShortTitle + teaserSubtitle + teaserText + tag + publicationDate + teaserImage { + ...ImageFragment + __typename + } + teaserLink { + ...Link + __typename + } + teamEmblems { + ...TeamEmblems + __typename + } + teaserAppendix { + ... on GalleryTeaserAppendix { + galleryImages { + ...ImageFragment + __typename + } + __typename + } + ... on VideoTeaserAppendix { + documentId + kalturaId + __typename + } + __typename + } + markers + adMarker + channelId + } + fragment Link on Link { + __typename + newTabOrWindow + title + target { + ... on ExternalLink { + url + __typename + } + ... on InternalLink { + id + path + channelId + channelPath + differentChannelDomain + locale + pageType + __typename + } + __typename + } + } + fragment ImageFragment on ImageFragment { + __typename + url + alt + caption + copyright + origWidth + origHeight + aspectRatioImageOverrides { + ...AspectRatioImageOverride + __typename + } + } + fragment AspectRatioImageOverride on AspectRatioImageOverride { + aspectRatio + url + __typename + } + fragment TeamEmblems on TeamEmblems { + ownTeam + emblem { + ...ImageFragment + __typename + } + __typename + } +`; + +export const route: Route = { + path: '/news/:language?', + categories: ['new-media'], + example: '/fcbayern/news', + parameters: { + language: { + description: 'Language', + options: [ + { value: 'en', label: 'English' }, + { value: 'es', label: 'Español' }, + { value: 'de', label: 'Deutsch' }, + { value: 'zh', label: '中文' }, + ], + default: 'en', + }, + }, + radar: [ + { + source: ['fcbayern.com/:language/news', 'fcbayern.com/:language'], + target: '/news/:language', + }, + { + source: ['www.fcbayern.cn/news', 'www.fcbayern.cn'], + target: '/news/zh', + }, + ], + name: 'News', + maintainers: ['TonyRL'], + handler, + url: 'fcbayern.com', +}; + +async function handler(ctx: Context) { + const { language = 'en' } = ctx.req.param(); + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')!, 10) : 20; + + if (language === 'zh') { + return cnNewsHander(limit); + } + + const baseUrl = 'https://fcbayern.com'; + const channelId = languages[language] ?? languages.en; + + const response = await ofetch(`${baseUrl}/graphql`, { + method: 'POST', + query: { + client: 'fcbwebsite', + }, + body: { + operationName: 'Web_NewsSearch', + variables: { + query: '', + filterTags: ['tag_club', 'tag_team_women', 'tag_team_campus', 'tag_article_news', 'tag_competitions_bundesliga', 'tag_competitions_champions-league', 'tag_column_saebener-stories', 'tag_club_magazin-51'], + offset: 0, + count: limit, + channelId, + filterSections: ['section_professionals', 'section_club', 'section_women', 'section_campus', 'section_aroundtheworld'], + filterTypes: ['fcbhippo:ArticleDocument', 'fcbhippo:ImageGalleryDocument', 'fcbhippo:TeaserDocument'], + }, + query, + }, + }); + + const items = response.data.newsSearch.results.map((item) => ({ + title: item.teaserTitle, + description: item.teaserText, + link: `${baseUrl}${item.teaserLink.target.path}`, + pubDate: item.publicationDate ? parseDate(item.publicationDate) : undefined, + image: item.teaserImage?.url, + category: item.tag ? [item.tag] : undefined, + })); + + return { + title: 'FC Bayern München - News', + link: `${baseUrl}/${language}/news`, + language, + image: `${baseUrl}/favicon.ico`, + item: items, + }; +} diff --git a/lib/routes/gameapps/index.tsx b/lib/routes/gameapps/index.tsx index 24a4b29ff60e..1b2b25c3502e 100644 --- a/lib/routes/gameapps/index.tsx +++ b/lib/routes/gameapps/index.tsx @@ -37,8 +37,11 @@ async function handler() { const $ = load(response); item.title = $('meta[property="og:title"]').attr('content') ?? $('.news-title h1').text(); + item.category = $('.tags-wrap .tag-item') + .toArray() + .map((el) => $(el).text().trim().replace(/^#/, '')); - $('.pages').remove(); + $('.pages, .article-ad, .social-actions, .news-footer').remove(); // remove unwanted key value delete item.content; diff --git a/lib/routes/iapp/namespace.ts b/lib/routes/iapp/namespace.ts new file mode 100644 index 000000000000..e4c3fac1cc1c --- /dev/null +++ b/lib/routes/iapp/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'IAPP', + url: 'iapp.org', + lang: 'en', +}; diff --git a/lib/routes/iapp/news.ts b/lib/routes/iapp/news.ts new file mode 100644 index 000000000000..1af11e5d7d5e --- /dev/null +++ b/lib/routes/iapp/news.ts @@ -0,0 +1,101 @@ +import crypto from 'node:crypto'; + +import { load } from 'cheerio'; +import type { Context } from 'hono'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/news', + categories: ['new-media'], + example: '/iapp/news', + radar: [ + { + source: ['iapp.org/news'], + }, + ], + name: 'News', + maintainers: ['TonyRL'], + handler, + url: 'iapp.org/news', +}; + +async function handler(ctx: Context) { + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')!, 10) : 20; + const baseUrl = 'https://iapp.org'; + const link = `${baseUrl}/news`; + + const { appId, apiKey, description } = await cache.tryGet('iapp:algolia-credentials', async () => { + const html = await ofetch(link); + const $ = load(html); + let appId: string | undefined; + let apiKey: string | undefined; + $('script').each((_, el) => { + const text = $(el).text(); + const match = text.match(/\\"appID\\":\\"(\w+)\\",\\"apiKey\\":\\"(\w+)\\"/); + if (match) { + appId = match[1]; + apiKey = match[2]; + return false; + } + }); + if (!appId || !apiKey) { + throw new Error('Failed to extract Algolia credentials from iapp.org'); + } + return { + appId, + apiKey, + description: $('meta[name="description"]').attr('content'), + }; + }); + + const response = await ofetch(`https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries`, { + method: 'POST', + query: { + 'x-algolia-api-key': apiKey, + 'x-algolia-application-id': appId, + }, + headers: { + Referer: `${baseUrl}/`, + }, + body: { + requests: [ + { + indexName: 'all_article_date_desc', + clickAnalytics: true, + facets: ['news_tags.domains.domains', 'news_tags.industry.industry', 'news_tags.law_and_regulation.law_and_regulation', 'news_tags.subject.subject'], + filters: '_content_type:news_article', + highlightPostTag: '__/ais-highlight__', + highlightPreTag: '__ais-highlight__', + hitsPerPage: limit, + maxValuesPerFacet: 200, + page: 0, + query: '', + userToken: `anonymous-${crypto.randomUUID()}`, + }, + ], + }, + }); + + const items = response.results[0].hits.map((hit) => ({ + title: hit.article_details.headline, + description: hit.article_body?.map((block) => block.rte?.rich_text_editor).join('') || hit.entry_summary, + link: new URL(hit.url, baseUrl).href, + pubDate: parseDate(hit.article_details.date), + author: hit.article_details.author?.map((a) => a.title).join(', '), + category: [...(hit.news_tags?.domains?.domains || []), ...(hit.news_tags?.subject?.subject || []), ...(hit.tags_group?.internal_class || [])], + image: hit.main_image?.url, + })); + + return { + title: 'IAPP - News', + description, + link, + language: 'en' as const, + image: `${baseUrl}/favicon.ico`, + item: items, + }; +} diff --git a/lib/routes/infoq/utils.ts b/lib/routes/infoq/utils.ts index 7e09f3575286..c862eb48f8a8 100644 --- a/lib/routes/infoq/utils.ts +++ b/lib/routes/infoq/utils.ts @@ -22,10 +22,11 @@ const ProcessFeed = async (list, cache) => { const author = data.author ? data.author.map((p) => p.nickname).join(',') : data.no_author; const category = [...e.topic.map((t) => t.name), ...e.label.map((l) => l.name)]; const content = data.content_url ? (await got(data.content_url)).body : data.content; + const description = addCoverToDescription(parseContent(content), data.article_cover); return { title: data.article_title, - description: parseContent(content), + description, pubDate: parseDate(e.publish_time, 'x'), category, author, @@ -97,6 +98,10 @@ const parseToSimpleTexts = (content) => return parseToSimpleText(i.content); }); +function addCoverToDescription(content, cover) { + return `

${content}`; +} + function parseContent(content) { const isRichContent = content.startsWith(`{"`); if (!isRichContent) { diff --git a/lib/routes/iwara/ranking.ts b/lib/routes/iwara/ranking.ts index b2969e31b05e..4ff2fa022e70 100644 --- a/lib/routes/iwara/ranking.ts +++ b/lib/routes/iwara/ranking.ts @@ -58,7 +58,7 @@ async function handler(ctx) { const items = await cache.tryGet( `iwara:ranking:${type}:${sort}:${rating}`, async () => { - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -83,7 +83,7 @@ async function handler(ctx) { pubDate: parseDate(item.createdAt), })); } finally { - await destory(); + await destroy(); } }, config.cache.routeExpire, diff --git a/lib/routes/iwara/subscriptions.ts b/lib/routes/iwara/subscriptions.ts index 77b058370362..3f44add670be 100644 --- a/lib/routes/iwara/subscriptions.ts +++ b/lib/routes/iwara/subscriptions.ts @@ -69,7 +69,7 @@ async function handler() { const username = config.iwara.username; const password = config.iwara.password; - const { page, destory } = await getPuppeteerPage(rootUrl, { + const { page, destroy } = await getPuppeteerPage(rootUrl, { gotoConfig: { waitUntil: 'domcontentloaded', }, @@ -113,7 +113,10 @@ async function handler() { async () => { const result = await fetchApi(`${apiqRootUrl}/user/token`, { method: 'POST', - headers: { ...apiHeaders, Authorization: refreshHeaders.authorization }, + headers: { + ...apiHeaders, + Authorization: refreshHeaders.authorization, + }, }); return { authorization: 'Bearer ' + result.accessToken }; }, @@ -121,7 +124,10 @@ async function handler() { false ); - const authedHeaders = { ...apiHeaders, Authorization: authHeaders.authorization }; + const authedHeaders = { + ...apiHeaders, + Authorization: authHeaders.authorization, + }; // fetch subscriptions const [videoResponse, imageResponse] = await Promise.all([ @@ -177,7 +183,9 @@ async function handler() { } const apiUrl = item.link.replace('www.iwara.tv', 'apiq.iwara.tv'); - const response = await fetchApi(apiUrl, { headers: authedHeaders }); + const response = await fetchApi(apiUrl, { + headers: authedHeaders, + }); description = renderSubscriptionImages(response.files ? response.files.filter((f) => f.type === 'image').map((f) => `${imageRootUrl}/image/original/${f.id}/${f.name}`) : [item.imageUrl]); @@ -202,6 +210,6 @@ async function handler() { item: items, }; } finally { - await destory(); + await destroy(); } } diff --git a/lib/routes/jable/index.ts b/lib/routes/jable/index.ts new file mode 100644 index 000000000000..361fc240f066 --- /dev/null +++ b/lib/routes/jable/index.ts @@ -0,0 +1,99 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import got from '@/utils/got'; + +export const route: Route = { + path: '/search/:query', + categories: ['multimedia'], + example: '/jable/search/みなみ羽琉', + parameters: { + query: 'Search keyword', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + nsfw: true, + }, + radar: [ + { + source: ['jable.tv/search/:query'], + target: '/search/:query', + }, + ], + name: 'Jable 搜索结果', + maintainers: ['eve2ptp'], + handler, +}; + +interface VideoThumb { + title: string; + link: string; + thumb: string; + preview: string; +} + +function renderDescription(video: VideoThumb): string { + return `${video.title}`.trim(); +} + +async function handler(ctx) { + const { query } = ctx.req.param(); + + const params = new URLSearchParams({ + mode: 'async', + function: 'get_block', + block_id: 'list_videos_videos_list_search_result', + q: query, + sort_by: 'post_date', + }); + + const encodedQuery = encodeURIComponent(query); + const baseUrl = `https://jable.tv/search/${encodedQuery}/`; + const apiUrl = `${baseUrl}?${params.toString()}`; + + const response = await got(apiUrl); + const $ = load(response.data); + + const videos: VideoThumb[] = $('.video-img-box') + .toArray() + .map((el) => { + const $el = $(el); + const $titleLink = $el.find('.detail h6.title a'); + const $img = $el.find('img'); + + return { + title: $titleLink.text().trim(), + link: $titleLink.attr('href') ?? '', + thumb: $img.attr('data-src') ?? '', + preview: $img.attr('data-preview') ?? '', + }; + }); + + const items = videos.map((video) => ({ + title: video.title, + link: video.link, + author: query, + description: renderDescription(video), + media: { + content: { + url: video.preview || video.link, + type: 'video/mp4', + }, + thumbnail: { + url: video.thumb, + }, + }, + })); + + return { + title: `${query} - Search | Jable`, + link: baseUrl, + description: `Search results for ${query}`, + item: items, + }; +} diff --git a/lib/routes/jable/namaspace.ts b/lib/routes/jable/namaspace.ts new file mode 100644 index 000000000000..f0a3ace67e5d --- /dev/null +++ b/lib/routes/jable/namaspace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'jable', + url: 'jable.tv', + lang: 'zh', +}; diff --git a/lib/routes/javdb/utils.ts b/lib/routes/javdb/utils.ts index 133a22a148f2..983a99dea7d8 100644 --- a/lib/routes/javdb/utils.ts +++ b/lib/routes/javdb/utils.ts @@ -7,7 +7,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; -const allowDomain = new Set(['javdb.com', 'javdb36.com', 'javdb007.com', 'javdb521.com']); +const allowDomain = new Set(['javdb.com', 'javdb571.com', 'javdb36.com', 'javdb007.com', 'javdb521.com']); const ProcessItems = async (ctx, currentUrl, title) => { const domain = ctx.req.query('domain') ?? 'javdb.com'; diff --git a/lib/routes/nanhua/namespace.ts b/lib/routes/nanhua/namespace.ts new file mode 100644 index 000000000000..53dd19eee902 --- /dev/null +++ b/lib/routes/nanhua/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: '南华期货', + url: 'mall.nanhua.net', + lang: 'zh-CN', +}; diff --git a/lib/routes/nanhua/report.ts b/lib/routes/nanhua/report.ts new file mode 100644 index 000000000000..a34dcbac25a4 --- /dev/null +++ b/lib/routes/nanhua/report.ts @@ -0,0 +1,77 @@ +import type { Context } from 'hono'; + +import type { DataItem, Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; +import timezone from '@/utils/timezone'; + +export const route: Route = { + path: '/report/:type1/:type2', + categories: ['finance'], + example: '/nanhua/report/WEEK/WEEK_black', + parameters: { + type1: '一级分类代码，如 `WEEK`（周度报告）、`SEASON`（季年报告）、`HOT`（热点报告）等，需要使用 `encodeURIComponent` 编码', + type2: '二级分类代码，如 `WEEK_black`（黑色）、`WEEK_enchem`（能化）等，需要使用 `encodeURIComponent` 编码', + }, + radar: [ + { + source: ['mall.nanhua.net/mall/r/w/reportNew/report-list.html'], + target: (_, url) => { + const params = new URL(url).searchParams; + const type1 = params.get('type1'); + const type2 = params.get('type2'); + return type1 && type2 ? `/nanhua/report/${encodeURIComponent(type1)}/${encodeURIComponent(type2)}` : ''; + }, + }, + ], + name: '研报', + maintainers: ['TonyRL'], + handler, + url: 'mall.nanhua.net/mall/r/w/reportNew/report-list.html', +}; + +async function handler(ctx: Context) { + const { type1, type2 } = ctx.req.param(); + const baseUrl = 'https://mall.nanhua.net'; + const apiBase = `${baseUrl}/mall/nh/api`; + const link = `${baseUrl}/mall/r/w/reportNew/report-list.html?type1=${type1}&type2=${type2}`; + + const treeList = await cache.tryGet('nanhua:treeList', async () => { + const response = await ofetch(`${apiBase}/reportType/getTreeList.json`, { + method: 'POST', + body: {}, + }); + return response.data; + }); + + const response = await ofetch(`${apiBase}/report/getPage.json`, { + method: 'POST', + body: { + type1Code: type1, + type2Code: type2, + pageSize: ctx.req.query('limit') || '20', + }, + }); + + const parent = treeList.find((item) => item.type === type1); + const child = parent?.children?.find((item) => item.type === type2); + + const items: DataItem[] = response.data.result.map((item) => ({ + title: item.title, + description: item.content || item.summary || item.desc, + link: item.fileName ? `${apiBase}/report/getReportFile?reportId=${item.id}&filetitle=${item.fileName}` : /^\d+$/.test(item.id) ? item.detailUrl : `${baseUrl}/mall/r/w/reportNew/report-list-page.html?id=${item.id}`, + pubDate: timezone(parseDate(item.createTime), 8), + author: item.personName, + category: [item.type1Name, item.type2Name, item.typeName].filter(Boolean), + image: item.iconUrl, + })); + + return { + title: `南华期货 - ${parent?.name ?? type1} - ${child?.name ?? type2}`, + link, + language: 'zh-CN' as const, + image: `${baseUrl}/favicon.ico`, + item: items, + }; +} diff --git a/lib/routes/nhentai/util.tsx b/lib/routes/nhentai/util.tsx index 16f84cd23507..50fc28432294 100644 --- a/lib/routes/nhentai/util.tsx +++ b/lib/routes/nhentai/util.tsx @@ -73,7 +73,7 @@ const fetchPage = async (url: string): Promise => { } catch (error: unknown) { const status = (error as { status?: number; statusCode?: number }).status ?? (error as { status?: number; statusCode?: number }).statusCode; if (status === 403) { - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { const allowedTypes = new Set(['document', 'script', 'xhr', 'fetch']); await page.setRequestInterception(true); @@ -83,7 +83,7 @@ const fetchPage = async (url: string): Promise => { }, }); const content = await page.content(); - await destory(); + await destroy(); return content; } throw error; diff --git a/lib/routes/perplexity/blog.ts b/lib/routes/perplexity/blog.ts index 98f3f61cd096..c55ebac1a57c 100644 --- a/lib/routes/perplexity/blog.ts +++ b/lib/routes/perplexity/blog.ts @@ -38,7 +38,7 @@ async function handler(ctx: Context) { const limit = Number.parseInt(ctx.req.query('limit') ?? '20', 10); const rootUrl = 'https://www.perplexity.ai/hub'; - const { page, destory, browser } = await getPuppeteerPage(rootUrl, { + const { page, destroy, browser } = await getPuppeteerPage(rootUrl, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -119,7 +119,9 @@ async function handler(ctx: Context) { request.resourceType() === 'document' ? request.continue() : request.abort(); }); - await contentPage.goto(item.link!, { waitUntil: 'domcontentloaded' }); + await contentPage.goto(item.link!, { + waitUntil: 'domcontentloaded', + }); const contentHtml = await contentPage.evaluate(() => document.documentElement.innerHTML); await contentPage.close(); @@ -148,7 +150,7 @@ async function handler(ctx: Context) { }) ); - await destory(); + await destroy(); return { title: 'Perplexity Blog', diff --git a/lib/routes/perplexity/changelog.ts b/lib/routes/perplexity/changelog.ts index b6dd5ecd0e21..9d4698e5e497 100644 --- a/lib/routes/perplexity/changelog.ts +++ b/lib/routes/perplexity/changelog.ts @@ -16,7 +16,7 @@ export const handler = async (ctx: Context): Promise => { logger.http(`Fetching Perplexity changelog from ${targetUrl}`); - const { page, destory, browser } = await getPuppeteerPage(targetUrl, { + const { page, destroy, browser } = await getPuppeteerPage(targetUrl, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -131,7 +131,7 @@ export const handler = async (ctx: Context): Promise => { ); // Close the browser session after all requests are done - await destory(); + await destroy(); return { title: $('title').text() || 'Perplexity Changelog', diff --git a/lib/routes/peterwunder/achievements.ts b/lib/routes/peterwunder/achievements.ts new file mode 100644 index 000000000000..3ba06d5d61c8 --- /dev/null +++ b/lib/routes/peterwunder/achievements.ts @@ -0,0 +1,138 @@ +import type { CheerioAPI } from 'cheerio'; +import { load } from 'cheerio'; + +import type { DataItem, Route } from '@/types'; +import { ViewType } from '@/types'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import { parseDate } from '@/utils/parse-date'; + +const author = 'Peter Wunder'; +const rootUrl = 'https://projects.peterwunder.de'; +const currentUrl = new URL('/achievements/', rootUrl).href; +const icon = new URL('/achievements/images/touchicon.png', rootUrl).href; +const defaultLimit = 20; + +type BadgeItem = DataItem & { + link: string; + title: string; +}; + +function absolutizeImageSource($: CheerioAPI, itemUrl: string) { + $('article') + .first() + .find('[src]') + .each((_, element) => { + const value = $(element).attr('src'); + + if (value) { + $(element).attr('src', new URL(value, itemUrl).href); + } + }); +} + +function extractBadgeDescription($: CheerioAPI) { + const article = $('article').first(); + + if (!article.length) { + return; + } + + article.find('h1, script, style, noscript').remove(); + + return article.html() ?? undefined; +} + +function extractListItems($: CheerioAPI, limit: number): BadgeItem[] { + return $('section.badges a.badge') + .slice(0, limit) + .toArray() + .map((element) => { + const badge = $(element); + const href = badge.attr('href'); + const title = badge.find('.title').text().trim(); + + if (!href || !title) { + return null; + } + + const image = badge.find('img').attr('src'); + const visibleStart = badge.attr('data-vis-start'); + + return { + title, + link: new URL(href, rootUrl).href, + pubDate: visibleStart ? parseDate(visibleStart) : undefined, + image: image ? new URL(image, rootUrl).href : undefined, + }; + }) + .filter(Boolean) as BadgeItem[]; +} + +function fetchBadge(item: BadgeItem) { + return cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $: CheerioAPI = load(response); + + const title = $('article h1').first().text().trim(); + const visibleStart = $('ul.metadata li').first().find('time.date').first().attr('datetime'); + const image = $('meta[property="og:image"]').attr('content'); + absolutizeImageSource($, item.link); + + return { + ...item, + title: title || item.title, + description: extractBadgeDescription($), + pubDate: visibleStart ? parseDate(visibleStart) : item.pubDate, + author, + image: image ? new URL(image, rootUrl).href : item.image, + }; + }); +} + +const handler: Route['handler'] = async (ctx) => { + const limit = Math.max(Number.parseInt(ctx.req.query('limit') ?? '', 10) || defaultLimit, 1); + + const { data: response } = await got(currentUrl); + const $: CheerioAPI = load(response); + + const items = await Promise.all(extractListItems($, limit).map((item) => fetchBadge(item))); + + return { + title: 'All Activity Challenges - New Badges', + description: "Latest badge pages from Peter Wunder's All Activity Challenges catalog. The website's own Atom feed was discontinued on August 20, 2024, so this route follows the latest entries directly from the site.", + link: currentUrl, + item: items, + language: 'en', + author, + icon, + logo: icon, + image: icon, + }; +}; + +export const route: Route = { + path: '/achievements', + categories: ['other'], + view: ViewType.Pictures, + example: '/peterwunder/achievements', + parameters: {}, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['projects.peterwunder.de/achievements'], + }, + ], + name: 'New Badges', + maintainers: ['LinxHex'], + description: "Latest badge pages from Peter Wunder's All Activity Challenges catalog. `pubDate` uses the first 'Visible in the app' date because the site does not expose a publication timestamp.", + handler, + url: 'projects.peterwunder.de/achievements', +}; diff --git a/lib/routes/peterwunder/namespace.ts b/lib/routes/peterwunder/namespace.ts new file mode 100644 index 000000000000..52e07a75e2fd --- /dev/null +++ b/lib/routes/peterwunder/namespace.ts @@ -0,0 +1,9 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Peter Wunder', + url: 'projects.peterwunder.de', + categories: ['other'], + description: 'Projects and catalogs maintained by Peter Wunder.', + lang: 'en', +}; diff --git a/lib/routes/picnob/utils.ts b/lib/routes/picnob/utils.ts index 94cda7c887c3..1a849f0c2ba1 100644 --- a/lib/routes/picnob/utils.ts +++ b/lib/routes/picnob/utils.ts @@ -2,7 +2,7 @@ import { getPuppeteerPage } from '@/utils/puppeteer'; const puppeteerGet = async (url) => { let data; - const { destory } = await getPuppeteerPage(url, { + const { destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -13,7 +13,7 @@ const puppeteerGet = async (url) => { }); }, }); - await destory(); + await destroy(); return data; }; diff --git a/lib/routes/picuki/profile.ts b/lib/routes/picuki/profile.ts index 92c1653fddec..17aadadc819f 100644 --- a/lib/routes/picuki/profile.ts +++ b/lib/routes/picuki/profile.ts @@ -85,7 +85,7 @@ async function handler(ctx) { }); } catch (error) { if (error.status === 403) { - const { page, destory } = await getPuppeteerPage(profileUrl, { + const { page, destroy } = await getPuppeteerPage(profileUrl, { onBeforeLoad: async (page) => { const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); await page.setRequestInterception(true); @@ -96,7 +96,7 @@ async function handler(ctx) { }); await page.waitForSelector('.content'); response = await page.content(); - await destory(); + await destroy(); } else { throw new NotFoundError(error.message); } diff --git a/lib/routes/polymarket/event.ts b/lib/routes/polymarket/event.ts new file mode 100644 index 000000000000..a107e710c919 --- /dev/null +++ b/lib/routes/polymarket/event.ts @@ -0,0 +1,64 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import type { Event } from './types'; +import { GAMMA_API } from './types'; +import { formatOddsDisplay } from './utils'; + +export const route: Route = { + path: '/event/:slug', + categories: ['finance'], + example: '/polymarket/event/presidential-election-winner-2024', + parameters: { + slug: 'Event slug from the URL (e.g. presidential-election-winner-2024)', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['polymarket.com/event/:slug'], + target: '/event/:slug', + }, + ], + name: 'Event', + url: 'polymarket.com', + maintainers: ['heqi201255'], + handler, +}; + +async function handler(ctx) { + const slug = ctx.req.param('slug'); + + const event = await ofetch(`${GAMMA_API}/events/slug/${slug}`); + + if (!event) { + throw new Error('Event not found'); + } + + const items = event.markets.map((market) => ({ + title: market.question, + description: ` +

Odds: ${formatOddsDisplay(market)}

+

Volume: $${Number(market.volume || 0).toLocaleString()}

+ ${market.oneDayPriceChange ? `

24h Change: ${(market.oneDayPriceChange * 100).toFixed(1)}%

` : ''} + ${market.image ? `${market.question}` : ''} + `, + link: `https://polymarket.com/event/${event.slug}`, + pubDate: market.startDate || event.startDate ? parseDate(market.startDate || event.startDate!) : undefined, + category: event.tags?.map((t) => t.label).filter(Boolean) as string[], + })); + + return { + title: event.title, + link: `https://polymarket.com/event/${event.slug}`, + item: items, + description: event.description, + }; +} diff --git a/lib/routes/polymarket/events.ts b/lib/routes/polymarket/events.ts new file mode 100644 index 000000000000..df8c726ea065 --- /dev/null +++ b/lib/routes/polymarket/events.ts @@ -0,0 +1,69 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import type { EventsPagination } from './types'; +import { GAMMA_API } from './types'; +import { formatEventDescription } from './utils'; + +export const route: Route = { + path: '/events/:tagSlug?', + categories: ['finance'], + example: '/polymarket/events', + parameters: { + tagSlug: 'Tag slug to filter events, e.g. politics, sports, crypto. Omit for all events.', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['polymarket.com', 'polymarket.com/:tagSlug'], + target: '/events/:tagSlug', + }, + ], + name: 'Events', + url: 'polymarket.com', + maintainers: ['heqi201255'], + handler, +}; + +async function handler(ctx) { + const tagSlug = ctx.req.param('tagSlug'); + const limit = 30; + + const query: Record = { + active: true, + closed: false, + limit, + order: 'volume', + ascending: false, + }; + + if (tagSlug) { + query.tag_slug = tagSlug; + } + + const response = await ofetch(`${GAMMA_API}/events/pagination`, { query }); + + const data = response.data || []; + + const items = data.map((event) => ({ + title: event.title, + description: formatEventDescription(event), + link: `https://polymarket.com/event/${event.slug}`, + pubDate: event.startDate ? parseDate(event.startDate) : undefined, + category: event.tags?.map((t) => t.label).filter(Boolean) as string[], + })); + + return { + title: `Polymarket Events${tagSlug ? ` - ${tagSlug}` : ''}`, + link: tagSlug ? `https://polymarket.com/${tagSlug}` : 'https://polymarket.com', + item: items, + }; +} diff --git a/lib/routes/polymarket/leaderboard.ts b/lib/routes/polymarket/leaderboard.ts new file mode 100644 index 000000000000..6bdef90b5e55 --- /dev/null +++ b/lib/routes/polymarket/leaderboard.ts @@ -0,0 +1,70 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; + +import type { LeaderboardEntry } from './types'; +import { DATA_API } from './types'; + +export const route: Route = { + path: '/leaderboard/:category?/:timePeriod?', + categories: ['finance'], + example: '/polymarket/leaderboard', + parameters: { + category: { + description: 'Market category: OVERALL, POLITICS, SPORTS, CRYPTO, CULTURE, MENTIONS, WEATHER, ECONOMICS, TECH, FINANCE', + default: 'OVERALL', + }, + timePeriod: { + description: 'Time period: DAY, WEEK, MONTH, ALL', + default: 'DAY', + }, + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'Leaderboard', + url: 'polymarket.com', + maintainers: ['heqi201255'], + handler, +}; + +async function handler(ctx) { + const category = ctx.req.param('category') || 'OVERALL'; + const timePeriod = ctx.req.param('timePeriod') || 'DAY'; + + const data = await ofetch(`${DATA_API}/v1/leaderboard`, { + query: { + category, + timePeriod, + orderBy: 'PNL', + limit: 50, + }, + }); + + const items = data.map((entry) => ({ + title: `#${entry.rank} ${entry.userName || entry.proxyWallet}`, + description: ` +

Rank: #${entry.rank}

+

PnL: $${Number(entry.pnl).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}

+

Volume: $${Number(entry.vol).toLocaleString()}

+ ${entry.xUsername ? `

X: @${entry.xUsername}

` : ''} + ${entry.verifiedBadge ? '

✅ Verified

' : ''} + ${entry.profileImage ? `${entry.userName || 'Trader'}` : ''} + `, + link: `https://polymarket.com/portfolio?address=${entry.proxyWallet}`, + author: entry.userName || entry.proxyWallet, + })); + + const categoryName = category.charAt(0).toUpperCase() + category.slice(1).toLowerCase(); + const periodName = timePeriod.charAt(0).toUpperCase() + timePeriod.slice(1).toLowerCase(); + + return { + title: `Polymarket Leaderboard - ${categoryName} (${periodName})`, + link: 'https://polymarket.com/leaderboard', + item: items, + }; +} diff --git a/lib/routes/polymarket/namespace.ts b/lib/routes/polymarket/namespace.ts new file mode 100644 index 000000000000..19454d16db35 --- /dev/null +++ b/lib/routes/polymarket/namespace.ts @@ -0,0 +1,8 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Polymarket', + url: 'polymarket.com', + description: `Polymarket is a prediction market platform where you can bet on real-world events.`, + lang: 'en', +}; diff --git a/lib/routes/polymarket/positions.ts b/lib/routes/polymarket/positions.ts new file mode 100644 index 000000000000..de9aa1b94aef --- /dev/null +++ b/lib/routes/polymarket/positions.ts @@ -0,0 +1,78 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; + +import type { Position, PublicProfile } from './types'; +import { DATA_API, GAMMA_API } from './types'; + +export const route: Route = { + path: '/positions/:address', + categories: ['finance'], + example: '/polymarket/positions/0x7c3db723f1d4d8cb9c550095203b686cb11e5c6b', + parameters: { + address: 'Wallet address (0x...)', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'User Positions', + url: 'polymarket.com', + maintainers: ['heqi201255'], + handler, +}; + +async function handler(ctx) { + const address = ctx.req.param('address'); + + // Fetch profile and positions + let profile: PublicProfile | null = null; + let positions: Position[] = []; + + try { + profile = await ofetch(`${GAMMA_API}/public-profile`, { + query: { address }, + }); + } catch { + // Profile not found, continue without it + } + + try { + positions = await ofetch(`${DATA_API}/positions`, { + query: { + user: address, + limit: 50, + sortBy: 'CURRENT', + sortDirection: 'DESC', + }, + }); + } catch { + // Positions not found, continue with empty array + } + + const displayName = profile?.name || profile?.pseudonym || address; + + const items = positions.map((pos) => ({ + title: pos.title || `Position #${pos.conditionId.slice(0, 8)}`, + description: ` +

Outcome: ${pos.outcome || `#${pos.outcomeIndex}`}

+

Size: ${Number(pos.size).toLocaleString()}

+

Avg Price: $${Number(pos.avgPrice).toFixed(4)}

+

Current Price: $${Number(pos.curPrice).toFixed(4)}

+

Current Value: $${Number(pos.currentValue).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}

+

PnL: ${pos.cashPnl >= 0 ? '+' : ''}$${Number(pos.cashPnl).toFixed(2)} (${pos.percentPnl >= 0 ? '+' : ''}${Number(pos.percentPnl).toFixed(1)}%)

+ ${pos.title || 'Position'} + `, + link: pos.eventSlug ? `https://polymarket.com/event/${pos.eventSlug}` : pos.slug ? `https://polymarket.com/event/${pos.slug}` : 'https://polymarket.com', + author: displayName, + })); + + return { + title: `Polymarket Positions - ${displayName}`, + link: `https://polymarket.com/portfolio?address=${address}`, + item: items, + }; +} diff --git a/lib/routes/polymarket/search.ts b/lib/routes/polymarket/search.ts new file mode 100644 index 000000000000..add5c719caad --- /dev/null +++ b/lib/routes/polymarket/search.ts @@ -0,0 +1,53 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import type { SearchResult } from './types'; +import { GAMMA_API } from './types'; +import { formatEventDescription } from './utils'; + +export const route: Route = { + path: '/search/:query', + categories: ['finance'], + example: '/polymarket/search/trump', + parameters: { + query: 'Search query', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'Search', + url: 'polymarket.com', + maintainers: ['heqi201255'], + handler, +}; + +async function handler(ctx) { + const query = ctx.req.param('query'); + + const data = await ofetch(`${GAMMA_API}/public-search`, { + query: { + q: query, + limit_per_type: 30, + }, + }); + + const items = (data.events || []).map((event) => ({ + title: event.title, + description: formatEventDescription(event), + link: `https://polymarket.com/event/${event.slug}`, + pubDate: event.startDate ? parseDate(event.startDate) : undefined, + category: event.tags?.map((t) => t.label).filter(Boolean) as string[], + })); + + return { + title: `Polymarket Search - ${query}`, + link: `https://polymarket.com/search?q=${encodeURIComponent(query)}`, + item: items, + }; +} diff --git a/lib/routes/polymarket/series.ts b/lib/routes/polymarket/series.ts new file mode 100644 index 000000000000..f1c77f023c16 --- /dev/null +++ b/lib/routes/polymarket/series.ts @@ -0,0 +1,100 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import type { Event, Series } from './types'; +import { GAMMA_API } from './types'; +import { formatEventDescription } from './utils'; + +export const route: Route = { + path: '/series/:slug?', + categories: ['finance'], + example: '/polymarket/series', + parameters: { + slug: { + description: 'Series slug, e.g. nfl, nba, mlb. If omitted, lists all series.', + default: 'all', + }, + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['polymarket.com/series/:slug'], + target: '/series/:slug', + }, + ], + name: 'Series', + url: 'polymarket.com', + maintainers: ['heqi201255'], + handler, +}; + +async function handler(ctx) { + const slug = ctx.req.param('slug'); + const limit = 20; + + if (slug) { + // Get specific series by slug + const data = await ofetch(`${GAMMA_API}/series`, { + query: { + slug, + limit: 1, + }, + }); + + if (!data.length) { + throw new Error('Series not found'); + } + + const series = data[0]; + const events = series.events || []; + + const items = events.map((event: Event) => ({ + title: event.title, + description: formatEventDescription(event), + link: `https://polymarket.com/event/${event.slug}`, + pubDate: event.startDate ? parseDate(event.startDate) : undefined, + category: event.tags?.map((t) => t.label).filter(Boolean) as string[], + })); + + return { + title: `Polymarket Series - ${series.title}`, + link: `https://polymarket.com/series/${series.slug}`, + item: items, + }; + } else { + // List all series + const data = await ofetch(`${GAMMA_API}/series`, { + query: { + limit, + order: 'volume', + ascending: false, + }, + }); + + const items = data.map((series) => ({ + title: series.title, + description: ` + ${series.description ? `

${series.description}

` : ''} +

Volume: $${Number(series.volume || 0).toLocaleString()}

+

Liquidity: $${Number(series.liquidity || 0).toLocaleString()}

+ ${series.image ? `${series.title}` : ''} + `, + link: `https://polymarket.com/series/${series.slug}`, + pubDate: series.createdAt ? parseDate(series.createdAt) : undefined, + })); + + return { + title: 'Polymarket - Series', + link: 'https://polymarket.com', + item: items, + }; + } +} diff --git a/lib/routes/polymarket/types.ts b/lib/routes/polymarket/types.ts new file mode 100644 index 000000000000..fac5dff2131f --- /dev/null +++ b/lib/routes/polymarket/types.ts @@ -0,0 +1,114 @@ +// API Constants +export const GAMMA_API = 'https://gamma-api.polymarket.com'; +export const DATA_API = 'https://data-api.polymarket.com'; + +// Common Interfaces + +export interface Market { + id: string; + question: string; + slug?: string; + outcomes?: string; + outcomePrices?: string; + volume?: string; + image?: string; + oneDayPriceChange?: number; + endDate?: string; + startDate?: string; +} + +export interface Event { + id: string; + title: string; + slug: string; + description?: string; + volume?: number; + image?: string; + startDate?: string; + endDate?: string; + liquidity?: number; + live?: boolean; + markets?: Market[]; + tags?: Array<{ label?: string }>; +} + +export interface Series { + id: string; + title: string; + slug: string; + description?: string; + image?: string; + volume?: number; + liquidity?: number; + startDate?: string; + createdAt?: string; + updatedAt?: string; + events?: Event[]; +} + +export interface PublicProfile { + name?: string; + pseudonym?: string; + bio?: string; + proxyWallet?: string; + profileImage?: string; +} + +export interface Position { + conditionId: string; + size: number; + avgPrice: number; + currentValue: number; + cashPnl: number; + percentPnl: number; + curPrice: number; + title?: string; + slug?: string; + eventSlug?: string; + outcome?: string; + outcomeIndex?: number; + icon: string; + endDate?: string; +} + +export interface Activity { + timestamp: number; + type: 'TRADE' | 'SPLIT' | 'MERGE' | 'REDEEM' | 'REWARD' | 'CONVERSION' | 'MAKER_REBATE'; + size?: number; + usdcSize?: number; + price?: number; + side?: 'BUY' | 'SELL'; + outcomeIndex?: number; + title?: string; + slug?: string; + eventSlug?: string; + outcome?: string; + icon?: string; + transactionHash?: string; + name?: string; + pseudonym?: string; +} + +export interface LeaderboardEntry { + rank: string; + proxyWallet: string; + userName?: string; + vol: number; + pnl: number; + profileImage?: string; + xUsername?: string; + verifiedBadge?: boolean; +} + +export interface EventsPagination { + data: Event[]; + pagination?: { + hasMore: boolean; + totalResults: number; + }; +} + +export interface SearchResult { + events?: Event[]; + tags?: Array<{ id: string; label: string; slug: string; event_count?: number }>; +} diff --git a/lib/routes/polymarket/user.ts b/lib/routes/polymarket/user.ts new file mode 100644 index 000000000000..c285c2961d03 --- /dev/null +++ b/lib/routes/polymarket/user.ts @@ -0,0 +1,94 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import type { Activity, PublicProfile } from './types'; +import { DATA_API, GAMMA_API } from './types'; + +export const route: Route = { + path: '/user/:address', + categories: ['finance'], + example: '/polymarket/user/0x7c3db723f1d4d8cb9c550095203b686cb11e5c6b', + parameters: { + address: 'Wallet address (0x...)', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'User Activity', + url: 'polymarket.com', + maintainers: ['heqi201255'], + handler, +}; + +async function handler(ctx) { + const address = ctx.req.param('address'); + + // Fetch profile and activity + let profile: PublicProfile | null = null; + let activity: Activity[] = []; + + try { + profile = await ofetch(`${GAMMA_API}/public-profile`, { + query: { address }, + }); + } catch { + // Profile not found, continue without it + } + + try { + activity = await ofetch(`${DATA_API}/activity`, { + query: { + user: address, + limit: 50, + sortBy: 'TIMESTAMP', + sortDirection: 'DESC', + }, + }); + } catch { + // Activity not found, continue with empty array + } + + const displayName = profile?.name || profile?.pseudonym || address; + + const items = activity.map((act) => { + const typeEmoji: Record = { + TRADE: '💱', + SPLIT: '✂️', + MERGE: '🔗', + REDEEM: '💰', + REWARD: '🎁', + CONVERSION: '🔄', + MAKER_REBATE: '💵', + }; + + const typeLabel = `${typeEmoji[act.type] || '📝'} ${act.type}`; + + return { + title: act.title ? `${act.title} - ${act.outcome || `Outcome ${act.outcomeIndex}`}` : typeLabel, + description: ` +

Type: ${typeLabel}

+ ${act.side ? `

Side: ${act.side}

` : ''} + ${act.price === undefined ? '' : `

Price: $${act.price.toFixed(4)}

`} + ${act.size === undefined ? '' : `

Size: ${act.size.toLocaleString()}

`} + ${act.usdcSize === undefined ? '' : `

USDC: $${act.usdcSize.toLocaleString()}

`} + ${act.icon ? `${act.title || 'Market'}` : ''} + `, + link: act.eventSlug ? `https://polymarket.com/event/${act.eventSlug}` : act.slug ? `https://polymarket.com/event/${act.slug}` : 'https://polymarket.com', + pubDate: parseDate(act.timestamp * 1000), + author: displayName, + }; + }); + + return { + title: `Polymarket User - ${displayName}`, + link: `https://polymarket.com/portfolio?address=${address}`, + item: items, + description: profile?.bio || undefined, + }; +} diff --git a/lib/routes/polymarket/utils.ts b/lib/routes/polymarket/utils.ts new file mode 100644 index 000000000000..4af4b5a8835a --- /dev/null +++ b/lib/routes/polymarket/utils.ts @@ -0,0 +1,21 @@ +import type { Event, Market } from './types'; + +export function formatOddsDisplay(market: Market): string { + const outcomes = market.outcomes ? JSON.parse(market.outcomes) : []; + const prices = market.outcomePrices ? JSON.parse(market.outcomePrices) : []; + return prices.length > 0 ? outcomes.map((o: string, i: number) => `${o}: ${(Number(prices[i]) * 100).toFixed(1)}%`).join(' | ') : outcomes.join(' | ') || 'N/A'; +} + +export function formatEventDescription(event: Event): string { + const marketsHtml = (event.markets || []) + .slice(0, 3) + .map((market) => `
  • ${market.question}
    ${formatOddsDisplay(market)}
    • `) + .join(''); + + return ` + ${event.description ? `

    ${event.description}

    ` : ''} +

    Volume: $${Number(event.volume || 0).toLocaleString()}

    + ${marketsHtml ? `

    Markets:

      ${marketsHtml}
    ` : ''} + ${event.image ? `${event.title}` : ''} + `; +} diff --git a/lib/routes/projectjav/actress.ts b/lib/routes/projectjav/actress.ts new file mode 100644 index 000000000000..4c9b9a94e7d7 --- /dev/null +++ b/lib/routes/projectjav/actress.ts @@ -0,0 +1,36 @@ +import type { Route } from '@/types'; + +import { processItems, rootUrl } from './utils'; + +export const route: Route = { + path: '/actress/:id', + categories: ['multimedia'], + example: '/projectjav/actress/rima-arai-22198', + parameters: { id: 'Actress ID or slug, can be found in the actress page URL' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + nsfw: true, + }, + radar: [ + { + source: ['projectjav.com/actress/:id'], + target: '/actress/:id', + }, + ], + name: 'Actress', + maintainers: ['Exat1979'], + handler, + url: 'projectjav.com/', + description: 'Fetches the latest movies from a specific actress page on ProjectJAV.', +}; + +async function handler(ctx) { + const id = ctx.req.param('id').replace(/\/$/, ''); + const currentUrl = `${rootUrl}/actress/${id}`; + return await processItems(currentUrl); +} diff --git a/lib/routes/projectjav/namespace.ts b/lib/routes/projectjav/namespace.ts new file mode 100644 index 000000000000..9581f6264ed0 --- /dev/null +++ b/lib/routes/projectjav/namespace.ts @@ -0,0 +1,8 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'ProjectJAV', + url: 'projectjav.com', + description: 'ProjectJAV provides adult video content information and streaming.', + lang: 'en', +}; diff --git a/lib/routes/projectjav/utils.ts b/lib/routes/projectjav/utils.ts new file mode 100644 index 000000000000..d863ebc7bebd --- /dev/null +++ b/lib/routes/projectjav/utils.ts @@ -0,0 +1,97 @@ +import { load } from 'cheerio'; + +import type { DataItem } from '@/types'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import { parseDate } from '@/utils/parse-date'; + +const rootUrl = 'https://projectjav.com'; +const processItems = async (currentUrl: string) => { + const response = await got({ + method: 'get', + url: currentUrl, + }); + + const $ = load(response.data); + + let items: DataItem[] = $('div.video-item') + .toArray() + .map((element) => { + const item = $(element); + const link = item.find('a').attr('href'); + return { + title: item.find('div.name span').text() || '', + link: link?.startsWith('http') ? link : `${rootUrl}${link}`, + }; + }) + .filter((item) => item.link && /\/movie\/.*/.test(item.link)); + + items = await Promise.all( + items.map((item) => + cache.tryGet(item.link!, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + + const content = load(detailResponse.data); + + // Remove ads + content('div.top-banner-ads, div.bottom-content-ads').remove(); + + // Get main content + const mainContent = content('main'); + + // Extract title + const h1Title = mainContent.find('h1').text().trim(); + if (h1Title) { + item.title = h1Title; + } + + // Extract categories + const categories = mainContent + .find('div.badge a') + .toArray() + .map((v) => content(v).text().trim()) + .filter(Boolean); + if (categories.length > 0) { + item.category = categories; + } + + // Extract author/actress (support multiple actresses) + const actresses = mainContent + .find('div.actress-item a') + .toArray() + .map((v) => content(v).text().trim()) + .filter(Boolean); + if (actresses.length > 0) { + item.author = actresses.join(', '); + } + + // Extract date + const dateEl = mainContent + .find('div.second-main~div.row>div.col-3') + .toArray() + .find((el) => content(el).text().includes('Date added')); + const dateText = dateEl ? content(dateEl).next().text().trim() : null; + if (dateText) { + // ProjectJAV uses DD/MM/YYYY format + item.pubDate = parseDate(dateText, 'DD/MM/YYYY'); + } + + // Get description + item.description = mainContent.html() || ''; + + return item; + }) + ) + ); + + return { + title: $('title').text() || 'ProjectJAV', + link: currentUrl, + item: items, + }; +}; + +export { processItems, rootUrl }; diff --git a/lib/routes/qq/news/user.ts b/lib/routes/qq/news/user.ts index 191542514e2a..aad163a1c138 100644 --- a/lib/routes/qq/news/user.ts +++ b/lib/routes/qq/news/user.ts @@ -64,10 +64,12 @@ async function handler(ctx) { news = await Promise.all( response.newslist.map((item) => cache.tryGet(item.id, async () => { - const description = - item.articletype === '0' - ? load(await ofetch(`https://news.qq.com/rain/a/${item.id}`))('.rich_media_content').html()! - : `

    ${item.abstract}

    文章包含非文本内容，请在浏览器中打开查看

    `; + let description = `

    ${item.abstract}

    文章包含非文本内容，请在浏览器中打开查看

    `; + if (item.articletype === '0') { + const article = await ofetch(`https://news.qq.com/rain/a/${item.id}`); + const $ = load(article); + description = $('.rich_media_content').html()!; + } return { title: item.longtitle, description, diff --git a/lib/routes/telecompaper/news.ts b/lib/routes/telecompaper/news.ts index 04038c90df5a..822e9f4c74af 100644 --- a/lib/routes/telecompaper/news.ts +++ b/lib/routes/telecompaper/news.ts @@ -1,5 +1,4 @@ import { load } from 'cheerio'; -import FormData from 'form-data'; import { CookieJar } from 'tough-cookie'; import type { Route } from '@/types'; @@ -58,7 +57,7 @@ async function handler(ctx) { form.append('__EVENTTARGET', 'ctl00$MainPlaceHolder$ddlContentType'); form.append('__EVENTARGUMENT', ''); form.append('__LASTFOCUS', ''); - form.append('__VIEWSTATE', $('#__VIEWSTATE').attr('value')); + form.append('__VIEWSTATE', $('#__VIEWSTATE').attr('value') ?? ''); form.append('__VIEWSTATEGENERATOR', 'E4EF4CD1'); form.append('ctl00$header$searchText', ctx.req.param('keyword') || ''); form.append('ctl00$header$searchTextMobile', ctx.req.param('keyword') || ''); @@ -66,26 +65,26 @@ async function handler(ctx) { form.append( 'ctl00$MainPlaceHolder$ddlYears', year && year !== 'all' - ? $('select[name="ctl00$MainPlaceHolder$ddlYears"] option') + ? ($('select[name="ctl00$MainPlaceHolder$ddlYears"] option') .filter((index, element) => $(element).text().split(' (')[0] === ctx.req.param('year')) - .attr('value') + .attr('value') ?? '0') : '0' ); form.append( 'ctl00$MainPlaceHolder$ddlCountries', country && country !== 'all' - ? $('select[name="ctl00$MainPlaceHolder$ddlCountries"] option') + ? ($('select[name="ctl00$MainPlaceHolder$ddlCountries"] option') .filter((index, element) => $(element).text().split(' (')[0] === country) - .attr('value') + .attr('value') ?? '0') : '0' ); } form.append( 'ctl00$MainPlaceHolder$ddlContentType', type && type !== 'all' - ? $('select[name="ctl00$MainPlaceHolder$ddlContentType"] option') + ? ($('select[name="ctl00$MainPlaceHolder$ddlContentType"] option') .filter((index, element) => $(element).text().split(' (')[0] === type) - .attr('value') + .attr('value') ?? '') : '' ); diff --git a/lib/routes/telegram/tglib/channel.ts b/lib/routes/telegram/tglib/channel.ts index 6863cd07fa40..1c63528bce2c 100644 --- a/lib/routes/telegram/tglib/channel.ts +++ b/lib/routes/telegram/tglib/channel.ts @@ -35,6 +35,26 @@ export async function getPollResults(client, message, m: Api.MessageMediaPoll) { return txt; } +export function withSearchParams(src: string, params: Record) { + const url = new URL(src); + for (const [key, value] of Object.entries(params)) { + url.searchParams.set(key, value); + } + return url.toString(); +} + +export function getMessageMediaUrl(requestUrl: string, username: string, messageId: number) { + const request = new URL(requestUrl); + const url = new URL(`/telegram/media/${username}/${messageId}`, request.origin); + for (const key of ['key', 'code']) { + const value = request.searchParams.get(key); + if (value) { + url.searchParams.set(key, value); + } + } + return url.toString(); +} + export function getMediaLink(src: string, m: Api.TypeMessageMedia) { const doc = getDocument(m); const mime = doc ? doc.mimeType : ''; @@ -44,7 +64,7 @@ export function getMediaLink(src: string, m: Api.TypeMessageMedia) { } if (doc && mime.startsWith('video/')) { const vid = (doc.attributes.find((t) => t instanceof Api.DocumentAttributeVideo) ?? { w: 1080, h: 720 }) as { w: number; h: number }; - return ``; + return ``; } if (doc && mime.startsWith('audio/')) { return `
    ${getAudioTitle(m)}
    `; @@ -56,7 +76,7 @@ export function getMediaLink(src: string, m: Api.TypeMessageMedia) { linkText = ''; // remove filename, it's only an animated sticker } if ((doc.thumbs?.length ?? 0) > 0) { - linkText = `
    ${linkText}
    `; + linkText = `
    ${linkText}
    `; } return `${linkText}`; } @@ -155,7 +175,7 @@ export default async function handler(ctx: Context) { } // messages that have no text are shown as if they're one post // because in TG only 1 attachment per message is possible - const src = `${new URL(ctx.req.url).origin}/telegram/media/${username}/${message.id}`; + const src = getMessageMediaUrl(ctx.req.url, username, message.id); attachments.push(getMediaLink(src, media)); } if (message.replyMarkup instanceof Api.ReplyInlineMarkup) { diff --git a/lib/routes/thepaper/sidebar.ts b/lib/routes/thepaper/sidebar.ts index 3b3bdedc5c59..86b12b97ae8e 100644 --- a/lib/routes/thepaper/sidebar.ts +++ b/lib/routes/thepaper/sidebar.ts @@ -1,12 +1,13 @@ import type { Route } from '@/types'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import utils from './utils'; const sections = { hotNews: '澎湃热榜', - financialInformationNews: '澎湃财讯', + financialInformationNews: '澎湃快讯', morningEveningNews: '早晚报', + editorHandpicked: '要闻精选', }; export const route: Route = { @@ -20,7 +21,12 @@ export const route: Route = { name: '侧边栏', categories: ['new-media'], example: '/thepaper/sidebar', - parameters: { sec: '侧边栏 id，可选 `hotNews` 即 澎湃热榜、`financialInformationNews` 即 澎湃财讯、`morningEveningNews` 即 早晚报，默认为 `hotNews`' }, + parameters: { + sec: { + description: '侧边栏 id', + options: Object.entries(sections).map(([key, value]) => ({ label: value, value: key })), + }, + }, maintainers: ['bigfei'], handler, url: 'thepaper.cn/', @@ -29,14 +35,13 @@ export const route: Route = { async function handler(ctx) { const { sec = 'hotNews' } = ctx.req.param(); - const sidebar_url = `https://cache.thepaper.cn/contentapi/wwwIndex/rightSidebar`; - const sidebar_url_resp = await got(sidebar_url); - const sidebar_url_data = sidebar_url_resp.data; - const list = sidebar_url_data.data[sec]; + const sidebarUrl = 'https://cache.thepaper.cn/contentapi/wwwIndex/rightSidebar'; + const sidebarUrlData = await ofetch(sidebarUrl); + const list = sidebarUrlData.data[sec]; const items = await Promise.all(list.filter((item) => item.contId).map((item) => utils.ProcessItem(item, ctx))); return { - title: `澎湃新闻 - ${sections[sec]}`, + title: `澎湃新闻 - ${sections[sec] ?? sec}`, item: items, link: 'https://www.thepaper.cn', }; diff --git a/lib/routes/thinkingmachines/namespace.ts b/lib/routes/thinkingmachines/namespace.ts new file mode 100644 index 000000000000..6fdeafc03c48 --- /dev/null +++ b/lib/routes/thinkingmachines/namespace.ts @@ -0,0 +1,6 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Thinking Machines Lab', + url: 'thinkingmachines.ai', +}; diff --git a/lib/routes/thinkingmachines/news.ts b/lib/routes/thinkingmachines/news.ts new file mode 100644 index 000000000000..f9b7ae6f2eee --- /dev/null +++ b/lib/routes/thinkingmachines/news.ts @@ -0,0 +1,77 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/news', + name: 'News', + url: 'thinkingmachines.ai/news', + maintainers: ['w3nhao'], + example: '/thinkingmachines/news', + categories: ['programming'], + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + }, + radar: [ + { + source: ['thinkingmachines.ai/news', 'thinkingmachines.ai/news/'], + target: '/news', + }, + ], + handler, +}; + +async function handler() { + const baseUrl = 'https://thinkingmachines.ai'; + const listUrl = `${baseUrl}/news/`; + + const response = await ofetch(listUrl); + const $ = load(response); + + const items = $('main li a') + .toArray() + .map((el) => { + const $el = $(el); + const title = $el.find('.post-title').text().trim(); + const dateStr = $el.find('time.desktop-time').text().trim(); + const href = $el.attr('href') || ''; + const link = href.startsWith('http') ? href : `${baseUrl}${href}`; + + return { title, dateStr, link }; + }) + .filter((item) => item.title && item.link); + + const fullItems = await Promise.all( + items.map((item) => + cache.tryGet(item.link, async () => { + const articleResponse = await ofetch(item.link); + const $article = load(articleResponse); + + // Remove non-content elements + $article('nav, footer, header, script, style').remove(); + // Remove heading (title, author, pubDate) and paginator + $article('.post-heading, #post-prev-link, #post-next-link').remove(); + + const description = $article('main').html()?.trim() || ''; + + return { + title: item.title, + link: item.link, + pubDate: parseDate(item.dateStr, 'MMM D, YYYY'), + description, + }; + }) + ) + ); + + return { + title: 'Thinking Machines Lab - News', + link: listUrl, + item: fullItems, + }; +} diff --git a/lib/routes/trendforce/namespace.ts b/lib/routes/trendforce/namespace.ts index b52d0094a236..78b1bf4296c7 100644 --- a/lib/routes/trendforce/namespace.ts +++ b/lib/routes/trendforce/namespace.ts @@ -6,4 +6,8 @@ export const namespace: Namespace = { categories: ['new-media'], description: '', lang: 'en', + zh: { + name: '集邦咨询', + url: 'trendforce.cn', + }, }; diff --git a/lib/routes/trendforce/news-cn.ts b/lib/routes/trendforce/news-cn.ts new file mode 100644 index 000000000000..b10979cea88f --- /dev/null +++ b/lib/routes/trendforce/news-cn.ts @@ -0,0 +1,77 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/cn/presscenter/news', + categories: ['new-media'], + example: '/trendforce/cn/presscenter/news', + radar: [ + { + source: ['www.trendforce.cn/presscenter/news'], + }, + ], + name: '产业洞察', + maintainers: ['TonyRL'], + handler, + url: 'www.trendforce.cn/presscenter/news', +}; + +async function handler() { + const baseUrl = 'https://www.trendforce.cn'; + const link = `${baseUrl}/presscenter/news`; + + const response = await ofetch(link); + const $ = load(response); + + const list = $('.list-items .list-item') + .toArray() + .map((item) => { + const $item = $(item); + const a = $item.find('h3 a.title-link'); + return { + title: a.find('strong').text().trim(), + link: new URL(a.attr('href')!, baseUrl).href, + description: $item.find('p').text()?.trim(), + pubDate: parseDate($item.find('h4').text().trim()), + }; + }); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const response = await ofetch(item.link); + const $ = load(response); + + const pressCenter = $('.presscenter'); + const tagRow = pressCenter.find('.tag-row'); + + item.category = tagRow + .find('.fa-bookmark') + .parent() + .find('a') + .toArray() + .map((a) => $(a).text().trim()); + item.author = tagRow.find('.fa-user').parent().find('a').text().trim(); + + pressCenter.find('h1, .tag-row, .press-choose-post, hr').remove(); + + item.description = pressCenter.html()?.trim(); + + return item; + }) + ) + ); + + return { + title: $('head title').text().trim(), + description: $('meta[name="description"]').attr('content'), + link, + language: $('html').attr('lang'), + image: `${baseUrl}${$('link[rel="apple-touch-icon-precomposed"][sizes="152x152"]').attr('href')}`, + item: items, + }; +} diff --git a/lib/routes/twitter/api/web-api/api.ts b/lib/routes/twitter/api/web-api/api.ts index be9c908cc42b..b412517badef 100644 --- a/lib/routes/twitter/api/web-api/api.ts +++ b/lib/routes/twitter/api/web-api/api.ts @@ -3,7 +3,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter'; import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; -import { baseUrl, gqlFeatures, gqlMap } from './constants'; +import { baseUrl, gqlFeatures, gqlMap, initGqlMap } from './constants'; import { gatherLegacyFromData, paginationTweets, twitterGot } from './utils'; const getUserData = (id) => @@ -216,5 +216,5 @@ export default { getList, getHomeTimeline, getHomeLatestTimeline, - init: () => {}, + init: initGqlMap, }; diff --git a/lib/routes/twitter/api/web-api/constants.ts b/lib/routes/twitter/api/web-api/constants.ts index b1ca1d46a1af..ea58af273d5e 100644 --- a/lib/routes/twitter/api/web-api/constants.ts +++ b/lib/routes/twitter/api/web-api/constants.ts @@ -1,19 +1,14 @@ +import { buildGqlMap, fallbackIds, resolveQueryIds } from './gql-id-resolver'; + const baseUrl = 'https://x.com/i/api'; -const graphQLEndpointsPlain = [ - '/graphql/E3opETHurmVJflFsUBVuUQ/UserTweets', - '/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName', - '/graphql/HJFjzBgCs16TqxewQOeLNg/HomeTimeline', - '/graphql/DiTkXJgLqBBxCs7zaYsbtA/HomeLatestTimeline', - '/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies', - '/graphql/dexO_2tohK86JDudXXG3Yw/UserMedia', - '/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId', - '/graphql/UN1i3zUiCWa-6r-Uaho4fw/SearchTimeline', - '/graphql/Pa45JvqZuKcW1plybfgBlQ/ListLatestTweetsTimeline', - '/graphql/QuBlQ6SxNAQCt6-kBiCXCQ/TweetDetail', -]; +// Initial gqlMap from fallback IDs, updated dynamically via initGqlMap() +let gqlMap: Record = buildGqlMap(fallbackIds); -const gqlMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3].replace(/V2$|Query$|QueryV2$/, ''), endpoint])); +const initGqlMap = async () => { + const queryIds = await resolveQueryIds(); + gqlMap = buildGqlMap(queryIds); +}; const thirdPartySupportedAPI = ['UserByScreenName', 'UserByRestId', 'UserTweets', 'UserTweetsAndReplies', 'ListLatestTweetsTimeline', 'SearchTimeline', 'UserMedia']; @@ -114,4 +109,4 @@ const timelineParams = { const bearerToken = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; -export { baseUrl, bearerToken, gqlFeatures, gqlMap, thirdPartySupportedAPI, timelineParams }; +export { baseUrl, bearerToken, gqlFeatures, gqlMap, initGqlMap, thirdPartySupportedAPI, timelineParams }; diff --git a/lib/routes/twitter/api/web-api/gql-id-resolver.ts b/lib/routes/twitter/api/web-api/gql-id-resolver.ts new file mode 100644 index 000000000000..9c2067aceba7 --- /dev/null +++ b/lib/routes/twitter/api/web-api/gql-id-resolver.ts @@ -0,0 +1,116 @@ +import { config } from '@/config'; +import cache from '@/utils/cache'; +import logger from '@/utils/logger'; +import ofetch from '@/utils/ofetch'; + +const CACHE_KEY = 'twitter:gql-query-ids'; + +// Hardcoded fallback IDs (last known working values) +export const fallbackIds: Record = { + UserTweets: 'E3opETHurmVJflFsUBVuUQ', + UserByScreenName: 'Yka-W8dz7RaEuQNkroPkYw', + HomeTimeline: 'xhYBF94fPSp8ey64FfYXiA', + HomeLatestTimeline: '0vp2Au9doTKsbn2vIk48Dg', + UserTweetsAndReplies: 'bt4TKuFz4T7Ckk-VvQVSow', + UserMedia: 'dexO_2tohK86JDudXXG3Yw', + UserByRestId: 'Qw77dDjp9xCpUY-AXwt-yQ', + SearchTimeline: 'UN1i3zUiCWa-6r-Uaho4fw', + ListLatestTweetsTimeline: 'Pa45JvqZuKcW1plybfgBlQ', + TweetDetail: 'QuBlQ6SxNAQCt6-kBiCXCQ', +}; + +const operationNames = Object.keys(fallbackIds); + +async function fetchTwitterPage(): Promise { + const response = await ofetch('https://x.com', { + parseResponse: (txt) => txt, + }); + return response as unknown as string; +} + +function extractQueryIds(scriptContent: string): Record { + const ids: Record = {}; + const matches = scriptContent.matchAll(/queryId:"([^"]+?)".+?operationName:"([^"]+?)"/g); + for (const match of matches) { + const [, queryId, operationName] = match; + if (operationNames.includes(operationName)) { + ids[operationName] = queryId; + } + } + return ids; +} + +async function fetchAndExtractIds(): Promise> { + const html = await fetchTwitterPage(); + + // Extract main.hash.js URL — it contains all the GraphQL query IDs we need + const mainMatch = html.match(/\/client-web\/main\.([a-z0-9]+)\./); + if (!mainMatch) { + logger.warn('twitter gql-id-resolver: main.js URL not found in Twitter page'); + return {}; + } + + const mainUrl = `https://abs.twimg.com/responsive-web/client-web/main.${mainMatch[1]}.js`; + logger.debug(`twitter gql-id-resolver: fetching ${mainUrl}`); + + const content = await ofetch(mainUrl, { + parseResponse: (txt) => txt, + }); + return extractQueryIds(content as unknown as string); +} + +let resolvePromise: Promise> | null = null; + +export async function resolveQueryIds(): Promise> { + // Check cache first + const cached = await cache.get(CACHE_KEY); + if (cached) { + try { + const parsed = typeof cached === 'string' ? JSON.parse(cached) : cached; + if (parsed && typeof parsed === 'object' && Object.keys(parsed).length > 0) { + logger.debug(`twitter gql-id-resolver: using cached query IDs`); + return { ...fallbackIds, ...parsed }; + } + } catch { + // ignore parse error + } + } + + // Deduplicate concurrent requests + if (!resolvePromise) { + resolvePromise = (async () => { + try { + logger.info('twitter gql-id-resolver: fetching fresh query IDs from Twitter JS bundles'); + const ids = await fetchAndExtractIds(); + + if (Object.keys(ids).length > 0) { + await cache.set(CACHE_KEY, JSON.stringify(ids), config.cache.contentExpire); + const found = operationNames.filter((name) => ids[name]); + const missing = operationNames.filter((name) => !ids[name]); + logger.debug(`twitter gql-id-resolver: resolved ${found.length}/${operationNames.length} query IDs. Missing: ${missing.join(', ') || 'none'}`); + } else { + logger.warn('twitter gql-id-resolver: failed to extract any query IDs, using fallback'); + } + + return ids; + } catch (error) { + logger.warn(`twitter gql-id-resolver: error fetching query IDs: ${error}. Using fallback.`); + return {}; + } finally { + resolvePromise = null; + } + })(); + } + + const ids = await resolvePromise; + return { ...fallbackIds, ...ids }; +} + +export function buildGqlMap(queryIds: Record): Record { + const map: Record = {}; + for (const name of operationNames) { + const id = queryIds[name] || fallbackIds[name]; + map[name] = `/graphql/${id}/${name}`; + } + return map; +} diff --git a/lib/routes/twitter/api/web-api/utils.ts b/lib/routes/twitter/api/web-api/utils.ts index becf3f74aaac..560211aabce4 100644 --- a/lib/routes/twitter/api/web-api/utils.ts +++ b/lib/routes/twitter/api/web-api/utils.ts @@ -1,7 +1,7 @@ import { cookie as HttpCookieAgentCookie, CookieAgent } from 'http-cookie-agent/undici'; import queryString from 'query-string'; import { Cookie, CookieJar } from 'tough-cookie'; -import { Client, ProxyAgent } from 'undici'; +import undici, { Client, ProxyAgent } from 'undici'; import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found'; @@ -136,8 +136,21 @@ export const twitterGot = async ( ) : {}; - const response = await ofetch.raw(requestUrl, { - retry: 0, + // Use undici.fetch directly instead of ofetch.raw to preserve the CookieAgent + // dispatcher. Two layers drop it in the normal path: + // 1. ofetch does not forward `dispatcher` to its internal fetch() call + // 2. wrappedFetch (request-rewriter) does `new Request(input, init)` which + // discards non-standard options like `dispatcher` + // Additionally, setting `cookie` header manually doesn't work either because + // the Fetch spec treats `cookie` as a forbidden header name, so + // `new Request()` silently strips it. + // The only way to send cookies via CookieAgent is to call undici.fetch with + // the dispatcher option directly. + // + // Because undici.fetch is the standard Fetch API and does not support ofetch's + // `onResponse` callback, the rate-limit and auth error handling that was + // previously in `onResponse` is now inlined below. + const response = await undici.fetch(requestUrl, { headers: { authority: 'x.com', accept: '*/*', @@ -160,67 +173,77 @@ export const twitterGot = async ( }), }, dispatcher: dispatchers?.agent, - onResponse: async ({ response }) => { - const remaining = response.headers.get('x-rate-limit-remaining'); - const remainingInt = Number.parseInt(remaining || '0'); - const reset = response.headers.get('x-rate-limit-reset'); - logger.debug( - `twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(response._data?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}` - ); - if (auth) { - if (remaining && remainingInt < 2 && reset) { - const resetTime = new Date(Number.parseInt(reset) * 1000); - const delay = (resetTime.getTime() - Date.now()) / 1000; - logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`); - await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2); - } else if (response.status === 429 || JSON.stringify(response._data?.data) === '{"user":{}}') { - logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`); - await cache.set(`${lockPrefix}${auth.token}`, '1', 2000); - } else if (response.status === 403 || response.status === 401) { - const newCookie = await login({ - username: auth.username, - password: auth.password, - authenticationSecret: auth.authenticationSecret, - }); - if (newCookie) { - logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`); - await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire); - logger.debug(`twitter debug: unlock twitter cookie for token ${auth.token} with error1`); - await cache.set(`${lockPrefix}${auth.token}`, '', 1); - } else { - const tokenIndex = config.twitter.authToken?.indexOf(auth.token); - if (tokenIndex !== undefined && tokenIndex !== -1) { - config.twitter.authToken?.splice(tokenIndex, 1); - } - if (auth.username) { - const usernameIndex = config.twitter.username?.indexOf(auth.username); - if (usernameIndex !== undefined && usernameIndex !== -1) { - config.twitter.username?.splice(usernameIndex, 1); - } - } - if (auth.password) { - const passwordIndex = config.twitter.password?.indexOf(auth.password); - if (passwordIndex !== undefined && passwordIndex !== -1) { - config.twitter.password?.splice(passwordIndex, 1); - } - } - logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`); - await cache.set(`${lockPrefix}${auth.token}`, '1', 3600); + }); + + let responseData: any; + try { + responseData = await response.json(); + } catch { + responseData = null; + } + + // Handle rate limiting and auth errors + const remaining = response.headers.get('x-rate-limit-remaining'); + const remainingInt = Number.parseInt(remaining || '0'); + const reset = response.headers.get('x-rate-limit-reset'); + logger.debug( + `twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(responseData?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}` + ); + if (auth) { + if (remaining && remainingInt < 2 && reset) { + const resetTime = new Date(Number.parseInt(reset) * 1000); + const delay = (resetTime.getTime() - Date.now()) / 1000; + logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`); + await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2); + } else if (response.status === 429 || JSON.stringify(responseData?.data) === '{"user":{}}') { + logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`); + await cache.set(`${lockPrefix}${auth.token}`, '1', 2000); + } else if (response.status === 403 || response.status === 401) { + const newCookie = await login({ + username: auth.username, + password: auth.password, + authenticationSecret: auth.authenticationSecret, + }); + if (newCookie) { + logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`); + await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire); + await cache.set(`${lockPrefix}${auth.token}`, '', 1); + } else { + const tokenIndex = config.twitter.authToken?.indexOf(auth.token); + if (tokenIndex !== undefined && tokenIndex !== -1) { + config.twitter.authToken?.splice(tokenIndex, 1); + } + if (auth.username) { + const usernameIndex = config.twitter.username?.indexOf(auth.username); + if (usernameIndex !== undefined && usernameIndex !== -1) { + config.twitter.username?.splice(usernameIndex, 1); + } + } + if (auth.password) { + const passwordIndex = config.twitter.password?.indexOf(auth.password); + if (passwordIndex !== undefined && passwordIndex !== -1) { + config.twitter.password?.splice(passwordIndex, 1); } - } else { - logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`); - await cache.set(`${lockPrefix}${auth.token}`, '', 1); } + logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`); + await cache.set(`${lockPrefix}${auth.token}`, '1', 3600); } - }, - }); + } else { + logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`); + await cache.set(`${lockPrefix}${auth.token}`, '', 1); + } + } + + if (response.status >= 400) { + throw new Error(`Twitter API error: ${response.status}`); + } if (auth?.token) { logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`); await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire); } - return response._data; + return responseData; }; export const paginationTweets = async (endpoint: string, userId: number | undefined, variables: Record, path?: string[]) => { diff --git a/lib/routes/wechat/wechat2rss.ts b/lib/routes/wechat/wechat2rss.ts index f518268c934e..a91b195fcfba 100644 --- a/lib/routes/wechat/wechat2rss.ts +++ b/lib/routes/wechat/wechat2rss.ts @@ -1,7 +1,6 @@ import type { Route } from '@/types'; import { parseDate } from '@/utils/parse-date'; import parser from '@/utils/rss-parser'; -import { finishArticleItem } from '@/utils/wechat-mp'; export const route: Route = { path: '/wechat2rss/:id', @@ -29,14 +28,13 @@ async function handler(ctx) { const { title, link, description, image, items: item } = await parser.parseURL(feedUrl); - let items = item.map((i) => ({ + const items = item.map((i) => ({ title: i.title, - pubDate: parseDate(i.pubDate), + pubDate: parseDate(i.isoDate), link: i.link, + description: i['content:encoded'] || i.content, })); - items = await Promise.all(items.map((item) => finishArticleItem(item))); - return { title, link, diff --git a/lib/routes/weibo/utils.ts b/lib/routes/weibo/utils.ts index 9750dd47bd07..f33e283157bb 100644 --- a/lib/routes/weibo/utils.ts +++ b/lib/routes/weibo/utils.ts @@ -80,7 +80,7 @@ const weiboUtils = { logger.info(`Fetching visitor Cookies from ${url}`); } let times = 0; - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { const expectResourceTypes = new Set(['document', 'script', 'xhr', 'fetch']); await page.setUserAgent(weiboUtils.apiHeaders['User-Agent']); @@ -101,7 +101,7 @@ const weiboUtils = { gotoConfig: { waitUntil: 'networkidle0' }, }); const cookies: string = await getCookies(page, 'weibo.cn'); - await destory(); + await destroy(); if (times < 2 || !cookies) { throw new Error(`Unable to fetch visitor cookies. Please set WEIBO_COOKIES. Redirection: ${times}, last URL: ${page.url()}`); } diff --git a/lib/routes/wkjyqh/namespace.ts b/lib/routes/wkjyqh/namespace.ts new file mode 100644 index 000000000000..9f25ff9f147c --- /dev/null +++ b/lib/routes/wkjyqh/namespace.ts @@ -0,0 +1,8 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: '五矿期货', + url: 'www.wkjyqh.com', + categories: ['finance'], + lang: 'zh-CN', +}; diff --git a/lib/routes/wkjyqh/research.ts b/lib/routes/wkjyqh/research.ts new file mode 100644 index 000000000000..31b6111a10d9 --- /dev/null +++ b/lib/routes/wkjyqh/research.ts @@ -0,0 +1,76 @@ +import { load } from 'cheerio'; + +import type { DataItem, Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; +import timezone from '@/utils/timezone'; + +export const route: Route = { + path: '/research', + categories: ['finance'], + example: '/wkjyqh/research', + radar: [ + { + source: ['www.wkjyqh.com/main/research_center/yjbg/index.shtml', 'www.wkjyqh.com/main/research_center/'], + }, + ], + name: '研究报告', + maintainers: ['TonyRL'], + handler, + url: 'www.wkjyqh.com/main/research_center/yjbg/index.shtml', +}; + +async function handler() { + const baseUrl = 'https://www.wkjyqh.com'; + const link = `${baseUrl}/main/research_center/yjbg/index.shtml`; + + const apiResponse = await ofetch(`${baseUrl}/servlet/json`, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + funcNo: '2000153', + catalogId: '232', + pageNow: '1', + pageSize: '15', + isFilter: '1', + titleLength: '35', + briefLength: '70', + _catalogId: '', + rightId: '', + }), + responseType: 'json', + }); + + const list: DataItem[] = apiResponse.results[0].data.map((item) => ({ + title: item.title, + link: new URL(item.url, baseUrl).href, + pubDate: timezone(parseDate(item.publish_date), +8), + })); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link!, async () => { + const response = await ofetch(item.link!); + const $ = load(response); + + const content = $('.article_detail'); + item.pubDate = timezone(parseDate(content.find('b#time').text()), +8); + content.find('h2, .tips').remove(); + item.description = content.html()?.trim(); + + return item; + }) + ) + ); + + return { + title: '五矿期货 - 研究报告', + link, + language: 'zh-CN' as const, + image: `${baseUrl}/favicon.ico`, + item: items, + }; +} diff --git a/lib/routes/xhamster/index.ts b/lib/routes/xhamster/index.ts new file mode 100644 index 000000000000..c92562d5a00c --- /dev/null +++ b/lib/routes/xhamster/index.ts @@ -0,0 +1,147 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import got from '@/utils/got'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/:creators', + categories: ['multimedia'], + example: '/xhamster/faustina-pierre', + parameters: { + creators: 'Creator slug from the URL (e.g. `faustina-pierre`)', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + nsfw: true, + }, + radar: [ + { + source: ['xhamster.com/creators/:creators', 'xhamster.com/creators/:creators/newest'], + target: '/:creators', + }, + ], + name: 'Newest Videos by Creator', + maintainers: ['eve2ptp'], + handler, + url: 'xhamster.com/faustina-pierre/newest', +}; + +interface VideoThumb { + id: number; + title: string; + pageURL: string; + thumbURL: string; + imageURL?: string; + trailerURL?: string; + trailerFallbackUrl?: string; + created?: number; + duration?: number; + views?: number; + isUHD?: boolean; +} + +interface Initials { + infoComponent?: { + pornstarTop?: { + name?: string; + }; + }; + trendingVideoSectionComponent?: { + videoListProps?: { + videoThumbProps?: VideoThumb[]; + }; + }; +} + +function extractInitials(scriptContent: string): Initials { + const match = scriptContent.match(/window\.initials\s*=\s*([\s\S]*?);?$/); + if (!match) { + throw new Error('initials not found'); + } + return JSON.parse(match[1]); +} + +function formatDuration(seconds: number): string { + const h = Math.floor(seconds / 3600); + const m = Math.floor((seconds % 3600) / 60); + const s = seconds % 60; + return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}` : `${m}:${String(s).padStart(2, '0')}`; +} + +function renderDescription(video: VideoThumb): string { + const thumb = video.imageURL ?? video.thumbURL; + const duration = video.duration ? formatDuration(video.duration) : ''; + const views = video.views === undefined ? '' : video.views.toLocaleString(); + const quality = video.isUHD ? '4K' : ''; + + let meta = ''; + + if (quality) { + meta += quality; + } + if (duration) { + meta += `${meta ? ' · ' : ''}Duration: ${duration}`; + } + if (views) { + meta += `${meta ? ' · ' : ''}Views: ${views}`; + } + + return ` + ${video.title} +

    ${meta}

    + `.trim(); +} + +async function handler(ctx) { + const { creators } = ctx.req.param(); + const pageUrl = `https://xhamster.com/creators/${encodeURIComponent(creators)}/newest`; + + const response = await got(pageUrl); + + const $ = load(response.data); + const initialsRaw = $('#initials-script').text(); + if (!initialsRaw) { + throw new Error('Could not locate initials script on page'); + } + + let initials: Initials; + try { + initials = extractInitials(initialsRaw); + } catch { + throw new Error('Failed to parse page data'); + } + + const creatorName = initials.infoComponent?.pornstarTop?.name ?? creators; + const videos = initials.trendingVideoSectionComponent?.videoListProps?.videoThumbProps ?? []; + + const items = videos.map((video) => ({ + title: `${video.title}${video.isUHD ? ' [4K]' : ''}`, + link: video.pageURL, + pubDate: video.created ? parseDate(video.created * 1000) : undefined, + author: creatorName, + description: renderDescription(video), + media: { + content: { + url: video.trailerURL ?? video.pageURL, + type: 'video/mp4', + ...(video.duration && { duration: video.duration }), + }, + thumbnail: { + url: video.imageURL ?? video.thumbURL, + }, + }, + })); + + return { + title: `${creatorName} - newest videos on xHamster`, + link: pageUrl, + description: `Latest videos from ${creatorName} on xHamster`, + item: items, + }; +} diff --git a/lib/routes/xhamster/namespace.ts b/lib/routes/xhamster/namespace.ts new file mode 100644 index 000000000000..48e5450af3b5 --- /dev/null +++ b/lib/routes/xhamster/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'xHamster', + url: 'xhamster.com', + lang: 'en', +}; diff --git a/lib/routes/xiaohongshu/user.ts b/lib/routes/xiaohongshu/user.ts index 138959c65cb5..5cc0d88f6009 100644 --- a/lib/routes/xiaohongshu/user.ts +++ b/lib/routes/xiaohongshu/user.ts @@ -103,7 +103,7 @@ async function getUserFeeds(url: string, category: string) { title: noteCard.displayTitle, link: new URL(noteCard.noteId || id, url).toString(), guid: noteCard.displayTitle, - description: `
    ${noteCard.displayTitle}`, + description: `
    ${noteCard.displayTitle}`, author: noteCard.user.nickname, upvotes: noteCard.interactInfo.likedCount, })) diff --git a/lib/routes/xiaohongshu/util.ts b/lib/routes/xiaohongshu/util.ts index b4247bea06ab..dcd23dd56e40 100644 --- a/lib/routes/xiaohongshu/util.ts +++ b/lib/routes/xiaohongshu/util.ts @@ -63,7 +63,7 @@ const getUser = (url, cache) => } // Use puppeteer - const { page, destory } = await getPuppeteerPage(url, { + const { page, destroy } = await getPuppeteerPage(url, { onBeforeLoad: async (page) => { await page.setRequestInterception(true); page.on('request', (request) => { @@ -107,7 +107,7 @@ const getUser = (url, cache) => return { userPageData, notes, collect }; } finally { - await destory(); + await destroy(); } }, config.cache.routeExpire, diff --git a/lib/utils/got.ts b/lib/utils/got.ts index deb9d7cbbf97..1c04dfb97072 100644 --- a/lib/utils/got.ts +++ b/lib/utils/got.ts @@ -5,7 +5,7 @@ import ofetch from '@/utils/ofetch'; import { getSearchParamsString } from './helpers'; const getFakeGot = (defaultOptions?: any) => { - const fakeGot = (request, options?: any) => { + const fakeGot = async (request, options?: any) => { if (!(typeof request === 'string' || request instanceof Request) && request.url) { options = { ...request, @@ -67,10 +67,11 @@ const getFakeGot = (defaultOptions?: any) => { const response = ofetch(request, options); if (options?.responseType === 'arrayBuffer') { - return response.then((responseData) => ({ + const responseData = await response; + return { data: Buffer.from(responseData), body: Buffer.from(responseData), - })); + }; } return response; }; diff --git a/lib/utils/puppeteer.mock.test.ts b/lib/utils/puppeteer.mock.test.ts index 713347ff3af0..a4c8e46de892 100644 --- a/lib/utils/puppeteer.mock.test.ts +++ b/lib/utils/puppeteer.mock.test.ts @@ -69,7 +69,7 @@ describe('getPuppeteerPage (mocked)', () => { expect(endpoint).toContain('stealth=true'); expect(onBeforeLoad).toHaveBeenCalled(); - await result.destory(); + await result.destroy(); expect(browser.close).toHaveBeenCalled(); delete process.env.PUPPETEER_WS_ENDPOINT; diff --git a/lib/utils/puppeteer.ts b/lib/utils/puppeteer.ts index 1abfbfc77351..2888aaf65f81 100644 --- a/lib/utils/puppeteer.ts +++ b/lib/utils/puppeteer.ts @@ -187,7 +187,7 @@ export const getPuppeteerPage = async ( return { page, - destory: async () => { + destroy: async () => { await browser.close(); }, browser, diff --git a/lib/utils/puppeteer.worker.ts b/lib/utils/puppeteer.worker.ts index 8fcc6efc090b..74f94425aad7 100644 --- a/lib/utils/puppeteer.worker.ts +++ b/lib/utils/puppeteer.worker.ts @@ -91,7 +91,7 @@ export const getPuppeteerPage = async ( return { page, - destory: async () => { + destroy: async () => { await browser.close(); }, browser, diff --git a/lib/utils/wechat-mp.test.ts b/lib/utils/wechat-mp.test.ts index e5e39325d70a..4e59d2b051f3 100644 --- a/lib/utils/wechat-mp.test.ts +++ b/lib/utils/wechat-mp.test.ts @@ -139,7 +139,7 @@ describe('wechat-mp', () => { // item_show_type in a separate script tag from real_item_show_type expect( ExtractMetadata.common( - load(/* HTML */ ` + load(`