yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 09:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(docs): throttle and retry requests in doc generation (#2847)

Browse Source
This commit is contained in:
Harshal Sheth 2021-07-08 15:17:51 -07:00 committed by GitHub
parent 799b0634e1
commit a117b63b55
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 51 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs-website/generateDocsDir.ts
View File

@ -3,6 +3,8 @@ import * as matter from "gray-matter";
import * as fs from "fs";
import * as path from "path";
import { Octokit } from "@octokit/rest";
import { throttling } from "@octokit/plugin-throttling";
import { retry } from "@octokit/plugin-retry";
// Note: this must be executed within the docs-website directory.
@ -18,7 +20,20 @@ const sidebars = require(SIDEBARS_DEF_PATH);
const sidebars_json = JSON.stringify(sidebars);
const sidebars_text = fs.readFileSync(SIDEBARS_DEF_PATH).toString();
const octokit = new Octokit({});
const MyOctokit = Octokit.plugin(retry).plugin(throttling);
const octokit = new MyOctokit({
throttle: {
onRateLimit: (retryAfter, options) => {
// Retry twice after rate limit is hit.
if (options.request.retryCount <= 2) {
return true;
}
},
onAbuseLimit: () => {
console.warn("GitHub API hit abuse limit");
},
},
});
function actually_in_sidebar(filepath: string): boolean {
const doc_id = get_id(filepath);

2
docs-website/package.json
View File

@ -18,6 +18,8 @@
"@docusaurus/core": "^2.0.0-beta.2",
"@docusaurus/plugin-ideal-image": "^2.0.0-beta.2",
"@docusaurus/preset-classic": "^2.0.0-beta.2",
"@octokit/plugin-retry": "^3.0.9",
"@octokit/plugin-throttling": "^3.5.1",
"@octokit/rest": "^18.6.2",
"clsx": "^1.1.1",
"react": "^16.12.0",

33
docs-website/yarn.lock
View File

@ -1754,6 +1754,11 @@
resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-7.3.5.tgz"
integrity sha512-6bm5lzGDOeSnWHM5W8OZ86RD2KpchynU+/Qlm5hNEFjfLDhwfAY2lSe68YRUEYFGlxSHe0HmakyhvmtWoD3Zog==
"@octokit/openapi-types@^8.2.1":
version "8.2.1"
resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-8.2.1.tgz#102e752a7378ff8d21057c70fd16f1c83856d8c5"
integrity sha512-BJz6kWuL3n+y+qM8Pv+UGbSxH6wxKf/SBs5yzGufMHwDefsa+Iq7ZGy1BINMD2z9SkXlIzk1qiu988rMuGXEMg==
"@octokit/plugin-paginate-rest@^2.6.2":
version "2.13.5"
resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.13.5.tgz"
@ -1774,6 +1779,22 @@
"@octokit/types" "^6.16.6"
deprecation "^2.3.1"
"@octokit/plugin-retry@^3.0.9":
version "3.0.9"
resolved "https://registry.yarnpkg.com/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz#ae625cca1e42b0253049102acd71c1d5134788fe"
integrity sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==
dependencies:
"@octokit/types" "^6.0.3"
bottleneck "^2.15.3"
"@octokit/plugin-throttling@^3.5.1":
version "3.5.1"
resolved "https://registry.yarnpkg.com/@octokit/plugin-throttling/-/plugin-throttling-3.5.1.tgz#889e80e7f3464a18c768337369498d2d47c0b643"
integrity sha512-d2jh3/RZo98DRw2J0jFxhKz7nrTGalGdkfRtxM+pI5k1wRb4BKBjiuE9cuZnhZyj+zLC1EcIptj7K+28LJZ3eA==
dependencies:
"@octokit/types" "^6.0.1"
bottleneck "^2.15.3"
"@octokit/request-error@^2.0.5", "@octokit/request-error@^2.1.0":
version "2.1.0"
resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-2.1.0.tgz"
@ -1805,6 +1826,13 @@
"@octokit/plugin-request-log" "^1.0.2"
"@octokit/plugin-rest-endpoint-methods" "5.3.3"
"@octokit/types@^6.0.1":
version "6.18.1"
resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.18.1.tgz#a6db178536e649fd5d67a7b747754bcc43940be4"
integrity sha512-5YsddjO1U+xC8ZYKV8yZYebW55PCc7qiEEeZ+wZRr6qyclynzfyD65KZ5FdtIeP0/cANyFaD7hV69qElf1nMsQ==
dependencies:
"@octokit/openapi-types" "^8.2.1"
"@octokit/types@^6.0.3", "@octokit/types@^6.13.0", "@octokit/types@^6.16.1", "@octokit/types@^6.16.6":
version "6.16.7"
resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.16.7.tgz"
@ -2681,6 +2709,11 @@ boolbase@^1.0.0, boolbase@~1.0.0:
resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz"
integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
bottleneck@^2.15.3:
version "2.19.5"
resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91"
integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==
boxen@^5.0.0, boxen@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.0.1.tgz"