diff --git a/apps/api/package.json b/apps/api/package.json index a0f9cf8e..dc62d4c3 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -53,7 +53,7 @@ "@bull-board/api": "^5.20.5", "@bull-board/express": "^5.20.5", "@devil7softwares/pos": "^1.0.2", - "@dqbd/tiktoken": "^1.0.13", + "@dqbd/tiktoken": "^1.0.17", "@hyperdx/node-opentelemetry": "^0.8.1", "@logtail/node": "^0.4.12", "@nangohq/node": "^0.40.8", @@ -73,6 +73,7 @@ "cacheable-lookup": "^6.1.0", "cheerio": "^1.0.0-rc.12", "cohere": "^1.1.1", + "cohere-ai": "^7.14.0", "cors": "^2.8.5", "cron-parser": "^4.9.0", "date-fns": "^3.6.0", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 095b507c..f07c4ecd 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -24,8 +24,8 @@ importers: specifier: ^1.0.2 version: 1.0.2 '@dqbd/tiktoken': - specifier: ^1.0.13 - version: 1.0.15 + specifier: ^1.0.17 + version: 1.0.17 '@hyperdx/node-opentelemetry': specifier: ^0.8.1 version: 0.8.1 @@ -83,6 +83,9 @@ importers: cohere: specifier: ^1.1.1 version: 1.1.1 + cohere-ai: + specifier: ^7.14.0 + version: 7.14.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)) cors: specifier: ^2.8.5 version: 2.8.5 @@ -130,7 +133,7 @@ importers: version: 2.9.0 langchain: specifier: ^0.2.8 - version: 0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0) + version: 0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0) languagedetect: specifier: ^2.0.0 version: 2.0.0 @@ -148,10 +151,10 @@ importers: version: 2.30.1 mongoose: specifier: ^8.4.4 - version: 8.4.4(socks@2.8.3) + version: 8.4.4(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3) natural: specifier: ^7.0.7 - version: 7.0.7(socks@2.8.3) + version: 7.0.7(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3) openai: specifier: ^4.57.0 version: 4.57.0(zod@3.23.8) @@ -289,6 +292,154 @@ packages: '@anthropic-ai/sdk@0.24.3': resolution: {integrity: sha512-916wJXO6T6k8R6BAAcLhLPv/pnLGy7YSEBZXZ1XTFbLcTZE8oTy3oDW9WJf9KKZwMvVcePIfoTSvzXHRcGxkQQ==} + '@aws-crypto/crc32@3.0.0': + resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==} + + '@aws-crypto/sha256-browser@5.2.0': + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + + '@aws-crypto/sha256-js@5.2.0': + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/supports-web-crypto@5.2.0': + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + + '@aws-crypto/util@3.0.0': + resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==} + + '@aws-crypto/util@5.2.0': + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + + '@aws-sdk/client-cognito-identity@3.679.0': + resolution: {integrity: sha512-vJzQ6QpaMu8itJMe3FH1/0rwMjL0ELh63iLTxiAmhiV/SvCwNNoSFLd2HdKxbV0Bg/x8lUiPVq3pl6+cxaIrEQ==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/client-sagemaker@3.679.0': + resolution: {integrity: sha512-n1hTHpQl6LwNkwn4vLmtbwkNoX2jxtiliRd0IaHR1CfAQvKNTfQ52mARWr73hR+/YcVsBzPx8sYKq2XHWArHKQ==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/client-sso-oidc@3.679.0': + resolution: {integrity: sha512-/dBYWcCwbA/id4sFCIVZvf0UsvzHCC68SryxeNQk/PDkY9N4n5yRcMUkZDaEyQCjowc3kY4JOXp2AdUP037nhA==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sts': ^3.679.0 + + '@aws-sdk/client-sso@3.679.0': + resolution: {integrity: sha512-/0cAvYnpOZTo/Y961F1kx2fhDDLUYZ0SQQ5/75gh3xVImLj7Zw+vp74ieqFbqWLYGMaq8z1Arr9A8zG95mbLdg==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/client-sts@3.679.0': + resolution: {integrity: sha512-3CvrT8w1RjFu1g8vKA5Azfr5V83r2/b68Ock43WE003Bq/5Y38mwmYX7vk0fPHzC3qejt4YMAWk/C3fSKOy25g==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/core@3.679.0': + resolution: {integrity: sha512-CS6PWGX8l4v/xyvX8RtXnBisdCa5+URzKd0L6GvHChype9qKUVxO/Gg6N/y43Hvg7MNWJt9FBPNWIxUB+byJwg==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/credential-provider-cognito-identity@3.679.0': + resolution: {integrity: sha512-XvWd6RPk7TA7tmqITT+NXvJ6ltJP8BUtLO1NAvja4HKExPKR9HAyoOeeH7KM3lVRED4e4LUnLb3fzteH20IXaA==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/credential-provider-env@3.679.0': + resolution: {integrity: sha512-EdlTYbzMm3G7VUNAMxr9S1nC1qUNqhKlAxFU8E7cKsAe8Bp29CD5HAs3POc56AVo9GC4yRIS+/mtlZSmrckzUA==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/credential-provider-http@3.679.0': + resolution: {integrity: sha512-ZoKLubW5DqqV1/2a3TSn+9sSKg0T8SsYMt1JeirnuLJF0mCoYFUaWMyvxxKuxPoqvUsaycxKru4GkpJ10ltNBw==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/credential-provider-ini@3.679.0': + resolution: {integrity: sha512-Rg7t8RwUzKcumpipG4neZqaeJ6DF+Bco1+FHn5BZB68jpvwvjBjcQUuWkxj18B6ctYHr1fkunnzeKEn/+vy7+w==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sts': ^3.679.0 + + '@aws-sdk/credential-provider-node@3.679.0': + resolution: {integrity: sha512-E3lBtaqCte8tWs6Rkssc8sLzvGoJ10TLGvpkijOlz43wPd6xCRh1YLwg6zolf9fVFtEyUs/GsgymiASOyxhFtw==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/credential-provider-process@3.679.0': + resolution: {integrity: sha512-u/p4TV8kQ0zJWDdZD4+vdQFTMhkDEJFws040Gm113VHa/Xo1SYOjbpvqeuFoz6VmM0bLvoOWjxB9MxnSQbwKpQ==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/credential-provider-sso@3.679.0': + resolution: {integrity: sha512-SAtWonhi9asxn0ukEbcE81jkyanKgqpsrtskvYPpO9Z9KOednM4Cqt6h1bfcS9zaHjN2zu815Gv8O7WiV+F/DQ==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.679.0': + resolution: {integrity: sha512-a74tLccVznXCaBefWPSysUcLXYJiSkeUmQGtalNgJ1vGkE36W5l/8czFiiowdWdKWz7+x6xf0w+Kjkjlj42Ung==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sts': ^3.679.0 + + '@aws-sdk/credential-providers@3.679.0': + resolution: {integrity: sha512-ZjZZb6OERw/UKlSqcJ24AUJIf/ekDLPZrPpo0kPMV70EQ0GkBiklIZ8qULu9bEcI2I4UIapBKRiXTrK4gA6YHg==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/middleware-host-header@3.679.0': + resolution: {integrity: sha512-y176HuQ8JRY3hGX8rQzHDSbCl9P5Ny9l16z4xmaiLo+Qfte7ee4Yr3yaAKd7GFoJ3/Mhud2XZ37fR015MfYl2w==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/middleware-logger@3.679.0': + resolution: {integrity: sha512-0vet8InEj7nvIvGKk+ch7bEF5SyZ7Us9U7YTEgXPrBNStKeRUsgwRm0ijPWWd0a3oz2okaEwXsFl7G/vI0XiEA==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/middleware-recursion-detection@3.679.0': + resolution: {integrity: sha512-sQoAZFsQiW/LL3DfKMYwBoGjYDEnMbA9WslWN8xneCmBAwKo6IcSksvYs23PP8XMIoBGe2I2J9BSr654XWygTQ==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/middleware-user-agent@3.679.0': + resolution: {integrity: sha512-4hdeXhPDURPqQLPd9jCpUEo9fQITXl3NM3W1MwcJpE0gdUM36uXkQOYsTPeeU/IRCLVjK8Htlh2oCaM9iJrLCA==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/protocol-http@3.374.0': + resolution: {integrity: sha512-9WpRUbINdGroV3HiZZIBoJvL2ndoWk39OfwxWs2otxByppJZNN14bg/lvCx5e8ggHUti7IBk5rb0nqQZ4m05pg==} + engines: {node: '>=14.0.0'} + deprecated: This package has moved to @smithy/protocol-http + + '@aws-sdk/region-config-resolver@3.679.0': + resolution: {integrity: sha512-Ybx54P8Tg6KKq5ck7uwdjiKif7n/8g1x+V0V9uTjBjRWqaIgiqzXwKWoPj6NCNkE7tJNtqI4JrNxp/3S3HvmRw==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/signature-v4@3.374.0': + resolution: {integrity: sha512-2xLJvSdzcZZAg0lsDLUAuSQuihzK0dcxIK7WmfuJeF7DGKJFmp9czQmz5f3qiDz6IDQzvgK1M9vtJSVCslJbyQ==} + engines: {node: '>=14.0.0'} + deprecated: This package has moved to @smithy/signature-v4 + + '@aws-sdk/token-providers@3.679.0': + resolution: {integrity: sha512-1/+Zso/x2jqgutKixYFQEGli0FELTgah6bm7aB+m2FAWH4Hz7+iMUsazg6nSWm714sG9G3h5u42Dmpvi9X6/hA==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sso-oidc': ^3.679.0 + + '@aws-sdk/types@3.679.0': + resolution: {integrity: sha512-NwVq8YvInxQdJ47+zz4fH3BRRLC6lL+WLkvr242PVBbUOLRyK/lkwHlfiKUoeVIMyK5NF+up6TRg71t/8Bny6Q==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/util-endpoints@3.679.0': + resolution: {integrity: sha512-YL6s4Y/1zC45OvddvgE139fjeWSKKPgLlnfrvhVL7alNyY9n7beR4uhoDpNrt5mI6sn9qiBF17790o+xLAXjjg==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/util-locate-window@3.679.0': + resolution: {integrity: sha512-zKTd48/ZWrCplkXpYDABI74rQlbR0DNHs8nH95htfSLj9/mWRSwaGptoxwcihaq/77vi/fl2X3y0a1Bo8bt7RA==} + engines: {node: '>=16.0.0'} + + '@aws-sdk/util-user-agent-browser@3.679.0': + resolution: {integrity: sha512-CusSm2bTBG1kFypcsqU8COhnYc6zltobsqs3nRrvYqYaOqtMnuE46K4XTWpnzKgwDejgZGOE+WYyprtAxrPvmQ==} + + '@aws-sdk/util-user-agent-node@3.679.0': + resolution: {integrity: sha512-Bw4uXZ+NU5ed6TNfo4tBbhBSW+2eQxXYjYBGl5gLUNUpg2pDFToQAP6rXBFiwcG52V2ny5oLGiD82SoYuYkAVg==} + engines: {node: '>=16.0.0'} + peerDependencies: + aws-crt: '>=1.0.0' + peerDependenciesMeta: + aws-crt: + optional: true + + '@aws-sdk/util-utf8-browser@3.259.0': + resolution: {integrity: sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==} + '@babel/code-frame@7.24.6': resolution: {integrity: sha512-ZJhac6FkEd1yhG2AHOmfcXG4ceoLltoCVJjN5XsWN9BifBQr+cHJbWi0h68HZuSORq+3WtJ2z0hwF2NG1b5kcA==} engines: {node: '>=6.9.0'} @@ -499,8 +650,8 @@ packages: engines: {node: '>=0'} deprecated: This package has been renamed to `fast-tag-pos` - '@dqbd/tiktoken@1.0.15': - resolution: {integrity: sha512-a6I67K1xUkuqcuwulobIJiLikkoE7egMaviI1Jg5bxSn2V7QGqXsGE3jTKr8UIOU/o74mAAd5TkeXFNBtaKF4A==} + '@dqbd/tiktoken@1.0.17': + resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==} '@flydotio/dockerfile@0.4.11': resolution: {integrity: sha512-L52UAfrOhmAn3T4TxpeRofQOSO+Kctg+uraB4nLzo4mvvh+4Z7HYxSi7Dnq0Kirz+xx6fDIc4OMNT1EdaORecA==} @@ -1475,6 +1626,216 @@ packages: '@sinonjs/fake-timers@10.3.0': resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==} + '@smithy/abort-controller@3.1.6': + resolution: {integrity: sha512-0XuhuHQlEqbNQZp7QxxrFTdVWdwxch4vjxYgfInF91hZFkPxf9QDrdQka0KfxFMPqLNzSw0b95uGTrLliQUavQ==} + engines: {node: '>=16.0.0'} + + '@smithy/config-resolver@3.0.10': + resolution: {integrity: sha512-Uh0Sz9gdUuz538nvkPiyv1DZRX9+D15EKDtnQP5rYVAzM/dnYk3P8cg73jcxyOitPgT3mE3OVj7ky7sibzHWkw==} + engines: {node: '>=16.0.0'} + + '@smithy/core@2.5.1': + resolution: {integrity: sha512-DujtuDA7BGEKExJ05W5OdxCoyekcKT3Rhg1ZGeiUWaz2BJIWXjZmsG/DIP4W48GHno7AQwRsaCb8NcBgH3QZpg==} + engines: {node: '>=16.0.0'} + + '@smithy/credential-provider-imds@3.2.5': + resolution: {integrity: sha512-4FTQGAsuwqTzVMmiRVTn0RR9GrbRfkP0wfu/tXWVHd2LgNpTY0uglQpIScXK4NaEyXbB3JmZt8gfVqO50lP8wg==} + engines: {node: '>=16.0.0'} + + '@smithy/eventstream-codec@1.1.0': + resolution: {integrity: sha512-3tEbUb8t8an226jKB6V/Q2XU/J53lCwCzULuBPEaF4JjSh+FlCMp7TmogE/Aij5J9DwlsZ4VAD/IRDuQ/0ZtMw==} + + '@smithy/fetch-http-handler@3.2.9': + resolution: {integrity: sha512-hYNVQOqhFQ6vOpenifFME546f0GfJn2OiQ3M0FDmuUu8V/Uiwy2wej7ZXxFBNqdx0R5DZAqWM1l6VRhGz8oE6A==} + + '@smithy/fetch-http-handler@4.0.0': + resolution: {integrity: sha512-MLb1f5tbBO2X6K4lMEKJvxeLooyg7guq48C2zKr4qM7F2Gpkz4dc+hdSgu77pCJ76jVqFBjZczHYAs6dp15N+g==} + + '@smithy/hash-node@3.0.8': + resolution: {integrity: sha512-tlNQYbfpWXHimHqrvgo14DrMAgUBua/cNoz9fMYcDmYej7MAmUcjav/QKQbFc3NrcPxeJ7QClER4tWZmfwoPng==} + engines: {node: '>=16.0.0'} + + '@smithy/invalid-dependency@3.0.8': + resolution: {integrity: sha512-7Qynk6NWtTQhnGTTZwks++nJhQ1O54Mzi7fz4PqZOiYXb4Z1Flpb2yRvdALoggTS8xjtohWUM+RygOtB30YL3Q==} + + '@smithy/is-array-buffer@1.1.0': + resolution: {integrity: sha512-twpQ/n+3OWZJ7Z+xu43MJErmhB/WO/mMTnqR6PwWQShvSJ/emx5d1N59LQZk6ZpTAeuRWrc+eHhkzTp9NFjNRQ==} + engines: {node: '>=14.0.0'} + + '@smithy/is-array-buffer@2.2.0': + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + + '@smithy/is-array-buffer@3.0.0': + resolution: {integrity: sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==} + engines: {node: '>=16.0.0'} + + '@smithy/middleware-content-length@3.0.10': + resolution: {integrity: sha512-T4dIdCs1d/+/qMpwhJ1DzOhxCZjZHbHazEPJWdB4GDi2HjIZllVzeBEcdJUN0fomV8DURsgOyrbEUzg3vzTaOg==} + engines: {node: '>=16.0.0'} + + '@smithy/middleware-endpoint@3.2.1': + resolution: {integrity: sha512-wWO3xYmFm6WRW8VsEJ5oU6h7aosFXfszlz3Dj176pTij6o21oZnzkCLzShfmRaaCHDkBXWBdO0c4sQAvLFP6zA==} + engines: {node: '>=16.0.0'} + + '@smithy/middleware-retry@3.0.25': + resolution: {integrity: sha512-m1F70cPaMBML4HiTgCw5I+jFNtjgz5z5UdGnUbG37vw6kh4UvizFYjqJGHvicfgKMkDL6mXwyPp5mhZg02g5sg==} + engines: {node: '>=16.0.0'} + + '@smithy/middleware-serde@3.0.8': + resolution: {integrity: sha512-Xg2jK9Wc/1g/MBMP/EUn2DLspN8LNt+GMe7cgF+Ty3vl+Zvu+VeZU5nmhveU+H8pxyTsjrAkci8NqY6OuvZnjA==} + engines: {node: '>=16.0.0'} + + '@smithy/middleware-stack@3.0.8': + resolution: {integrity: sha512-d7ZuwvYgp1+3682Nx0MD3D/HtkmZd49N3JUndYWQXfRZrYEnCWYc8BHcNmVsPAp9gKvlurdg/mubE6b/rPS9MA==} + engines: {node: '>=16.0.0'} + + '@smithy/node-config-provider@3.1.9': + resolution: {integrity: sha512-qRHoah49QJ71eemjuS/WhUXB+mpNtwHRWQr77J/m40ewBVVwvo52kYAmb7iuaECgGTTcYxHS4Wmewfwy++ueew==} + engines: {node: '>=16.0.0'} + + '@smithy/node-http-handler@3.2.5': + resolution: {integrity: sha512-PkOwPNeKdvX/jCpn0A8n9/TyoxjGZB8WVoJmm9YzsnAgggTj4CrjpRHlTQw7dlLZ320n1mY1y+nTRUDViKi/3w==} + engines: {node: '>=16.0.0'} + + '@smithy/property-provider@3.1.8': + resolution: {integrity: sha512-ukNUyo6rHmusG64lmkjFeXemwYuKge1BJ8CtpVKmrxQxc6rhUX0vebcptFA9MmrGsnLhwnnqeH83VTU9hwOpjA==} + engines: {node: '>=16.0.0'} + + '@smithy/protocol-http@1.2.0': + resolution: {integrity: sha512-GfGfruksi3nXdFok5RhgtOnWe5f6BndzYfmEXISD+5gAGdayFGpjWu5pIqIweTudMtse20bGbc+7MFZXT1Tb8Q==} + engines: {node: '>=14.0.0'} + + '@smithy/protocol-http@4.1.5': + resolution: {integrity: sha512-hsjtwpIemmCkm3ZV5fd/T0bPIugW1gJXwZ/hpuVubt2hEUApIoUTrf6qIdh9MAWlw0vjMrA1ztJLAwtNaZogvg==} + engines: {node: '>=16.0.0'} + + '@smithy/querystring-builder@3.0.8': + resolution: {integrity: sha512-btYxGVqFUARbUrN6VhL9c3dnSviIwBYD9Rz1jHuN1hgh28Fpv2xjU1HeCeDJX68xctz7r4l1PBnFhGg1WBBPuA==} + engines: {node: '>=16.0.0'} + + '@smithy/querystring-parser@3.0.8': + resolution: {integrity: sha512-BtEk3FG7Ks64GAbt+JnKqwuobJNX8VmFLBsKIwWr1D60T426fGrV2L3YS5siOcUhhp6/Y6yhBw1PSPxA5p7qGg==} + engines: {node: '>=16.0.0'} + + '@smithy/service-error-classification@3.0.8': + resolution: {integrity: sha512-uEC/kCCFto83bz5ZzapcrgGqHOh/0r69sZ2ZuHlgoD5kYgXJEThCoTuw/y1Ub3cE7aaKdznb+jD9xRPIfIwD7g==} + engines: {node: '>=16.0.0'} + + '@smithy/shared-ini-file-loader@3.1.9': + resolution: {integrity: sha512-/+OsJRNtoRbtsX0UpSgWVxFZLsJHo/4sTr+kBg/J78sr7iC+tHeOvOJrS5hCpVQ6sWBbhWLp1UNiuMyZhE6pmA==} + engines: {node: '>=16.0.0'} + + '@smithy/signature-v4@1.1.0': + resolution: {integrity: sha512-fDo3m7YqXBs7neciOePPd/X9LPm5QLlDMdIC4m1H6dgNLnXfLMFNIxEfPyohGA8VW9Wn4X8lygnPSGxDZSmp0Q==} + engines: {node: '>=14.0.0'} + + '@smithy/signature-v4@4.2.1': + resolution: {integrity: sha512-NsV1jF4EvmO5wqmaSzlnTVetemBS3FZHdyc5CExbDljcyJCEEkJr8ANu2JvtNbVg/9MvKAWV44kTrGS+Pi4INg==} + engines: {node: '>=16.0.0'} + + '@smithy/smithy-client@3.4.2': + resolution: {integrity: sha512-dxw1BDxJiY9/zI3cBqfVrInij6ShjpV4fmGHesGZZUiP9OSE/EVfdwdRz0PgvkEvrZHpsj2htRaHJfftE8giBA==} + engines: {node: '>=16.0.0'} + + '@smithy/types@1.2.0': + resolution: {integrity: sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==} + engines: {node: '>=14.0.0'} + + '@smithy/types@3.6.0': + resolution: {integrity: sha512-8VXK/KzOHefoC65yRgCn5vG1cysPJjHnOVt9d0ybFQSmJgQj152vMn4EkYhGuaOmnnZvCPav/KnYyE6/KsNZ2w==} + engines: {node: '>=16.0.0'} + + '@smithy/url-parser@3.0.8': + resolution: {integrity: sha512-4FdOhwpTW7jtSFWm7SpfLGKIBC9ZaTKG5nBF0wK24aoQKQyDIKUw3+KFWCQ9maMzrgTJIuOvOnsV2lLGW5XjTg==} + + '@smithy/util-base64@3.0.0': + resolution: {integrity: sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==} + engines: {node: '>=16.0.0'} + + '@smithy/util-body-length-browser@3.0.0': + resolution: {integrity: sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==} + + '@smithy/util-body-length-node@3.0.0': + resolution: {integrity: sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==} + engines: {node: '>=16.0.0'} + + '@smithy/util-buffer-from@1.1.0': + resolution: {integrity: sha512-9m6NXE0ww+ra5HKHCHig20T+FAwxBAm7DIdwc/767uGWbRcY720ybgPacQNB96JMOI7xVr/CDa3oMzKmW4a+kw==} + engines: {node: '>=14.0.0'} + + '@smithy/util-buffer-from@2.2.0': + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + + '@smithy/util-buffer-from@3.0.0': + resolution: {integrity: sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==} + engines: {node: '>=16.0.0'} + + '@smithy/util-config-provider@3.0.0': + resolution: {integrity: sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==} + engines: {node: '>=16.0.0'} + + '@smithy/util-defaults-mode-browser@3.0.25': + resolution: {integrity: sha512-fRw7zymjIDt6XxIsLwfJfYUfbGoO9CmCJk6rjJ/X5cd20+d2Is7xjU5Kt/AiDt6hX8DAf5dztmfP5O82gR9emA==} + engines: {node: '>= 10.0.0'} + + '@smithy/util-defaults-mode-node@3.0.25': + resolution: {integrity: sha512-H3BSZdBDiVZGzt8TG51Pd2FvFO0PAx/A0mJ0EH8a13KJ6iUCdYnw/Dk/MdC1kTd0eUuUGisDFaxXVXo4HHFL1g==} + engines: {node: '>= 10.0.0'} + + '@smithy/util-endpoints@2.1.4': + resolution: {integrity: sha512-kPt8j4emm7rdMWQyL0F89o92q10gvCUa6sBkBtDJ7nV2+P7wpXczzOfoDJ49CKXe5CCqb8dc1W+ZdLlrKzSAnQ==} + engines: {node: '>=16.0.0'} + + '@smithy/util-hex-encoding@1.1.0': + resolution: {integrity: sha512-7UtIE9eH0u41zpB60Jzr0oNCQ3hMJUabMcKRUVjmyHTXiWDE4vjSqN6qlih7rCNeKGbioS7f/y2Jgym4QZcKFg==} + engines: {node: '>=14.0.0'} + + '@smithy/util-hex-encoding@3.0.0': + resolution: {integrity: sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==} + engines: {node: '>=16.0.0'} + + '@smithy/util-middleware@1.1.0': + resolution: {integrity: sha512-6hhckcBqVgjWAqLy2vqlPZ3rfxLDhFWEmM7oLh2POGvsi7j0tHkbN7w4DFhuBExVJAbJ/qqxqZdRY6Fu7/OezQ==} + engines: {node: '>=14.0.0'} + + '@smithy/util-middleware@3.0.8': + resolution: {integrity: sha512-p7iYAPaQjoeM+AKABpYWeDdtwQNxasr4aXQEA/OmbOaug9V0odRVDy3Wx4ci8soljE/JXQo+abV0qZpW8NX0yA==} + engines: {node: '>=16.0.0'} + + '@smithy/util-retry@3.0.8': + resolution: {integrity: sha512-TCEhLnY581YJ+g1x0hapPz13JFqzmh/pMWL2KEFASC51qCfw3+Y47MrTmea4bUE5vsdxQ4F6/KFbUeSz22Q1ow==} + engines: {node: '>=16.0.0'} + + '@smithy/util-stream@3.2.1': + resolution: {integrity: sha512-R3ufuzJRxSJbE58K9AEnL/uSZyVdHzud9wLS8tIbXclxKzoe09CRohj2xV8wpx5tj7ZbiJaKYcutMm1eYgz/0A==} + engines: {node: '>=16.0.0'} + + '@smithy/util-uri-escape@1.1.0': + resolution: {integrity: sha512-/jL/V1xdVRt5XppwiaEU8Etp5WHZj609n0xMTuehmCqdoOFbId1M+aEeDWZsQ+8JbEB/BJ6ynY2SlYmOaKtt8w==} + engines: {node: '>=14.0.0'} + + '@smithy/util-uri-escape@3.0.0': + resolution: {integrity: sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==} + engines: {node: '>=16.0.0'} + + '@smithy/util-utf8@1.1.0': + resolution: {integrity: sha512-p/MYV+JmqmPyjdgyN2UxAeYDj9cBqCjp0C/NsTWnnjoZUVqoeZ6IrW915L9CAKWVECgv9lVQGc4u/yz26/bI1A==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@2.3.0': + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@3.0.0': + resolution: {integrity: sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==} + engines: {node: '>=16.0.0'} + + '@smithy/util-waiter@3.1.7': + resolution: {integrity: sha512-d5yGlQtmN/z5eoTtIYgkvOw27US2Ous4VycnXatyoImIF9tzlcpnKqQ/V7qhvJmb2p6xZne1NopCLakdTnkBBQ==} + engines: {node: '>=16.0.0'} + '@stdlib/assert-has-own-property@0.0.7': resolution: {integrity: sha512-3YHwSWiUqGlTLSwxAWxrqaD1PkgcJniGyotJeIt5X0tSNmSW0/c9RWroCImTUUB3zBkyBJ79MyU9Nf4Qgm59fQ==} engines: {node: '>=0.10.0', npm: '>2.7.0'} @@ -2004,6 +2365,9 @@ packages: bottleneck@2.19.5: resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==} + bowser@2.11.0: + resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==} + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -2146,6 +2510,9 @@ packages: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} + cohere-ai@7.14.0: + resolution: {integrity: sha512-hSo2/tFV29whjFFtVtdS7kHmtUsjfMO1sgwE/d5bhOE4O7Vkj5G1R9lLIqkIprp/+rrvCq3HGvEaOgry7xRcDA==} + cohere@1.1.1: resolution: {integrity: sha512-D116FKTuauCShJjPuOAFnkyAPMhV/6f403+yPZwyyFY6gErK1AA41y9rQdBvj8eHDZ9sXVJ6TzmzObVfAFh3ig==} @@ -2565,6 +2932,10 @@ packages: fast-safe-stringify@2.1.1: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + fast-xml-parser@4.4.1: + resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==} + hasBin: true + fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} @@ -2613,6 +2984,10 @@ packages: form-data-encoder@1.7.2: resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + form-data-encoder@4.0.2: + resolution: {integrity: sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==} + engines: {node: '>= 18'} + form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -2621,6 +2996,10 @@ packages: resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} engines: {node: '>= 12.20'} + formdata-node@6.0.3: + resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==} + engines: {node: '>= 18'} + formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -3107,6 +3486,9 @@ packages: joplin-turndown-plugin-gfm@1.0.12: resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==} + js-base64@3.7.2: + resolution: {integrity: sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==} + js-beautify@1.15.1: resolution: {integrity: sha512-ESjNzSlt/sWE8sciZH8kBF8BPlwXPwhR6pWKAw8bw4Bwj+iZcnKW6ONWUutJ7eObuBZQpiIb8S7OYspWrKt7rA==} engines: {node: '>=14'} @@ -4027,6 +4409,10 @@ packages: resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} engines: {node: '>=0.6'} + qs@6.11.2: + resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} + engines: {node: '>=0.6'} + qs@6.12.2: resolution: {integrity: sha512-x+NLUpx9SYrcwXtX7ob1gnkSems4i/mGZX5SlYxwIau6RrUSODO89TR/XDGGpn5RPWSYIB+aSfuSlV5+CmbTBg==} engines: {node: '>=0.6'} @@ -4365,6 +4751,9 @@ packages: resolution: {integrity: sha512-syeEEd112om/waJ5gOQ+SaYi+setuidQ4ZIPiQREF4yJeegXhn2HKy6C0JYm7uhVQKfMAvuZ22dIRsnoDv7AMw==} engines: {node: '>=12.*'} + strnum@1.0.5: + resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==} + supabase@1.172.2: resolution: {integrity: sha512-h2J6kKEikXnZyurUcCYg215qkQpINOhdWkiclHcWAuVeqXsNrfrYaf1s0qbbcdRyMtrVW48I+VdVTw71Cnn20Q==} engines: {npm: '>=8'} @@ -4526,8 +4915,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.6.2: - resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==} + typescript@5.6.3: + resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==} engines: {node: '>=14.17'} hasBin: true @@ -4797,6 +5186,503 @@ snapshots: transitivePeerDependencies: - encoding + '@aws-crypto/crc32@3.0.0': + dependencies: + '@aws-crypto/util': 3.0.0 + '@aws-sdk/types': 3.679.0 + tslib: 1.14.1 + + '@aws-crypto/sha256-browser@5.2.0': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-locate-window': 3.679.0 + '@smithy/util-utf8': 2.3.0 + tslib: 2.6.3 + + '@aws-crypto/sha256-js@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.679.0 + tslib: 2.6.3 + + '@aws-crypto/supports-web-crypto@5.2.0': + dependencies: + tslib: 2.6.3 + + '@aws-crypto/util@3.0.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-utf8-browser': 3.259.0 + tslib: 1.14.1 + + '@aws-crypto/util@5.2.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/util-utf8': 2.3.0 + tslib: 2.6.3 + + '@aws-sdk/client-cognito-identity@3.679.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/client-sts': 3.679.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/middleware-host-header': 3.679.0 + '@aws-sdk/middleware-logger': 3.679.0 + '@aws-sdk/middleware-recursion-detection': 3.679.0 + '@aws-sdk/middleware-user-agent': 3.679.0 + '@aws-sdk/region-config-resolver': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-endpoints': 3.679.0 + '@aws-sdk/util-user-agent-browser': 3.679.0 + '@aws-sdk/util-user-agent-node': 3.679.0 + '@smithy/config-resolver': 3.0.10 + '@smithy/core': 2.5.1 + '@smithy/fetch-http-handler': 3.2.9 + '@smithy/hash-node': 3.0.8 + '@smithy/invalid-dependency': 3.0.8 + '@smithy/middleware-content-length': 3.0.10 + '@smithy/middleware-endpoint': 3.2.1 + '@smithy/middleware-retry': 3.0.25 + '@smithy/middleware-serde': 3.0.8 + '@smithy/middleware-stack': 3.0.8 + '@smithy/node-config-provider': 3.1.9 + '@smithy/node-http-handler': 3.2.5 + '@smithy/protocol-http': 4.1.5 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/url-parser': 3.0.8 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.25 + '@smithy/util-defaults-mode-node': 3.0.25 + '@smithy/util-endpoints': 2.1.4 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-retry': 3.0.8 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/client-sagemaker@3.679.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/client-sts': 3.679.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/middleware-host-header': 3.679.0 + '@aws-sdk/middleware-logger': 3.679.0 + '@aws-sdk/middleware-recursion-detection': 3.679.0 + '@aws-sdk/middleware-user-agent': 3.679.0 + '@aws-sdk/region-config-resolver': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-endpoints': 3.679.0 + '@aws-sdk/util-user-agent-browser': 3.679.0 + '@aws-sdk/util-user-agent-node': 3.679.0 + '@smithy/config-resolver': 3.0.10 + '@smithy/core': 2.5.1 + '@smithy/fetch-http-handler': 3.2.9 + '@smithy/hash-node': 3.0.8 + '@smithy/invalid-dependency': 3.0.8 + '@smithy/middleware-content-length': 3.0.10 + '@smithy/middleware-endpoint': 3.2.1 + '@smithy/middleware-retry': 3.0.25 + '@smithy/middleware-serde': 3.0.8 + '@smithy/middleware-stack': 3.0.8 + '@smithy/node-config-provider': 3.1.9 + '@smithy/node-http-handler': 3.2.5 + '@smithy/protocol-http': 4.1.5 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/url-parser': 3.0.8 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.25 + '@smithy/util-defaults-mode-node': 3.0.25 + '@smithy/util-endpoints': 2.1.4 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-retry': 3.0.8 + '@smithy/util-utf8': 3.0.0 + '@smithy/util-waiter': 3.1.7 + '@types/uuid': 9.0.8 + tslib: 2.6.3 + uuid: 9.0.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sts': 3.679.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/middleware-host-header': 3.679.0 + '@aws-sdk/middleware-logger': 3.679.0 + '@aws-sdk/middleware-recursion-detection': 3.679.0 + '@aws-sdk/middleware-user-agent': 3.679.0 + '@aws-sdk/region-config-resolver': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-endpoints': 3.679.0 + '@aws-sdk/util-user-agent-browser': 3.679.0 + '@aws-sdk/util-user-agent-node': 3.679.0 + '@smithy/config-resolver': 3.0.10 + '@smithy/core': 2.5.1 + '@smithy/fetch-http-handler': 3.2.9 + '@smithy/hash-node': 3.0.8 + '@smithy/invalid-dependency': 3.0.8 + '@smithy/middleware-content-length': 3.0.10 + '@smithy/middleware-endpoint': 3.2.1 + '@smithy/middleware-retry': 3.0.25 + '@smithy/middleware-serde': 3.0.8 + '@smithy/middleware-stack': 3.0.8 + '@smithy/node-config-provider': 3.1.9 + '@smithy/node-http-handler': 3.2.5 + '@smithy/protocol-http': 4.1.5 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/url-parser': 3.0.8 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.25 + '@smithy/util-defaults-mode-node': 3.0.25 + '@smithy/util-endpoints': 2.1.4 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-retry': 3.0.8 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/client-sso@3.679.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/middleware-host-header': 3.679.0 + '@aws-sdk/middleware-logger': 3.679.0 + '@aws-sdk/middleware-recursion-detection': 3.679.0 + '@aws-sdk/middleware-user-agent': 3.679.0 + '@aws-sdk/region-config-resolver': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-endpoints': 3.679.0 + '@aws-sdk/util-user-agent-browser': 3.679.0 + '@aws-sdk/util-user-agent-node': 3.679.0 + '@smithy/config-resolver': 3.0.10 + '@smithy/core': 2.5.1 + '@smithy/fetch-http-handler': 3.2.9 + '@smithy/hash-node': 3.0.8 + '@smithy/invalid-dependency': 3.0.8 + '@smithy/middleware-content-length': 3.0.10 + '@smithy/middleware-endpoint': 3.2.1 + '@smithy/middleware-retry': 3.0.25 + '@smithy/middleware-serde': 3.0.8 + '@smithy/middleware-stack': 3.0.8 + '@smithy/node-config-provider': 3.1.9 + '@smithy/node-http-handler': 3.2.5 + '@smithy/protocol-http': 4.1.5 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/url-parser': 3.0.8 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.25 + '@smithy/util-defaults-mode-node': 3.0.25 + '@smithy/util-endpoints': 2.1.4 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-retry': 3.0.8 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/client-sts@3.679.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/core': 3.679.0 + '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/middleware-host-header': 3.679.0 + '@aws-sdk/middleware-logger': 3.679.0 + '@aws-sdk/middleware-recursion-detection': 3.679.0 + '@aws-sdk/middleware-user-agent': 3.679.0 + '@aws-sdk/region-config-resolver': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-endpoints': 3.679.0 + '@aws-sdk/util-user-agent-browser': 3.679.0 + '@aws-sdk/util-user-agent-node': 3.679.0 + '@smithy/config-resolver': 3.0.10 + '@smithy/core': 2.5.1 + '@smithy/fetch-http-handler': 3.2.9 + '@smithy/hash-node': 3.0.8 + '@smithy/invalid-dependency': 3.0.8 + '@smithy/middleware-content-length': 3.0.10 + '@smithy/middleware-endpoint': 3.2.1 + '@smithy/middleware-retry': 3.0.25 + '@smithy/middleware-serde': 3.0.8 + '@smithy/middleware-stack': 3.0.8 + '@smithy/node-config-provider': 3.1.9 + '@smithy/node-http-handler': 3.2.5 + '@smithy/protocol-http': 4.1.5 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/url-parser': 3.0.8 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.25 + '@smithy/util-defaults-mode-node': 3.0.25 + '@smithy/util-endpoints': 2.1.4 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-retry': 3.0.8 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/core@3.679.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/core': 2.5.1 + '@smithy/node-config-provider': 3.1.9 + '@smithy/property-provider': 3.1.8 + '@smithy/protocol-http': 4.1.5 + '@smithy/signature-v4': 4.2.1 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/util-middleware': 3.0.8 + fast-xml-parser: 4.4.1 + tslib: 2.6.3 + + '@aws-sdk/credential-provider-cognito-identity@3.679.0': + dependencies: + '@aws-sdk/client-cognito-identity': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@smithy/property-provider': 3.1.8 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-env@3.679.0': + dependencies: + '@aws-sdk/core': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@smithy/property-provider': 3.1.8 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/credential-provider-http@3.679.0': + dependencies: + '@aws-sdk/core': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@smithy/fetch-http-handler': 3.2.9 + '@smithy/node-http-handler': 3.2.5 + '@smithy/property-provider': 3.1.8 + '@smithy/protocol-http': 4.1.5 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/util-stream': 3.2.1 + tslib: 2.6.3 + + '@aws-sdk/credential-provider-ini@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0)': + dependencies: + '@aws-sdk/client-sts': 3.679.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/credential-provider-env': 3.679.0 + '@aws-sdk/credential-provider-http': 3.679.0 + '@aws-sdk/credential-provider-process': 3.679.0 + '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)) + '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/types': 3.679.0 + '@smithy/credential-provider-imds': 3.2.5 + '@smithy/property-provider': 3.1.8 + '@smithy/shared-ini-file-loader': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + + '@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0)': + dependencies: + '@aws-sdk/credential-provider-env': 3.679.0 + '@aws-sdk/credential-provider-http': 3.679.0 + '@aws-sdk/credential-provider-ini': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/credential-provider-process': 3.679.0 + '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)) + '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/types': 3.679.0 + '@smithy/credential-provider-imds': 3.2.5 + '@smithy/property-provider': 3.1.8 + '@smithy/shared-ini-file-loader': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - '@aws-sdk/client-sts' + - aws-crt + + '@aws-sdk/credential-provider-process@3.679.0': + dependencies: + '@aws-sdk/core': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@smithy/property-provider': 3.1.8 + '@smithy/shared-ini-file-loader': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/credential-provider-sso@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))': + dependencies: + '@aws-sdk/client-sso': 3.679.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/token-providers': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)) + '@aws-sdk/types': 3.679.0 + '@smithy/property-provider': 3.1.8 + '@smithy/shared-ini-file-loader': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + + '@aws-sdk/credential-provider-web-identity@3.679.0(@aws-sdk/client-sts@3.679.0)': + dependencies: + '@aws-sdk/client-sts': 3.679.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@smithy/property-provider': 3.1.8 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))': + dependencies: + '@aws-sdk/client-cognito-identity': 3.679.0 + '@aws-sdk/client-sso': 3.679.0 + '@aws-sdk/client-sts': 3.679.0 + '@aws-sdk/core': 3.679.0 + '@aws-sdk/credential-provider-cognito-identity': 3.679.0 + '@aws-sdk/credential-provider-env': 3.679.0 + '@aws-sdk/credential-provider-http': 3.679.0 + '@aws-sdk/credential-provider-ini': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/credential-provider-process': 3.679.0 + '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)) + '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/types': 3.679.0 + '@smithy/credential-provider-imds': 3.2.5 + '@smithy/property-provider': 3.1.8 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + + '@aws-sdk/middleware-host-header@3.679.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/protocol-http': 4.1.5 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/middleware-logger@3.679.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/middleware-recursion-detection@3.679.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/protocol-http': 4.1.5 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/middleware-user-agent@3.679.0': + dependencies: + '@aws-sdk/core': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@aws-sdk/util-endpoints': 3.679.0 + '@smithy/core': 2.5.1 + '@smithy/protocol-http': 4.1.5 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/protocol-http@3.374.0': + dependencies: + '@smithy/protocol-http': 1.2.0 + tslib: 2.6.3 + + '@aws-sdk/region-config-resolver@3.679.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/node-config-provider': 3.1.9 + '@smithy/types': 3.6.0 + '@smithy/util-config-provider': 3.0.0 + '@smithy/util-middleware': 3.0.8 + tslib: 2.6.3 + + '@aws-sdk/signature-v4@3.374.0': + dependencies: + '@smithy/signature-v4': 1.1.0 + tslib: 2.6.3 + + '@aws-sdk/token-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))': + dependencies: + '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/client-sts@3.679.0) + '@aws-sdk/types': 3.679.0 + '@smithy/property-provider': 3.1.8 + '@smithy/shared-ini-file-loader': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/types@3.679.0': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/util-endpoints@3.679.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/types': 3.6.0 + '@smithy/util-endpoints': 2.1.4 + tslib: 2.6.3 + + '@aws-sdk/util-locate-window@3.679.0': + dependencies: + tslib: 2.6.3 + + '@aws-sdk/util-user-agent-browser@3.679.0': + dependencies: + '@aws-sdk/types': 3.679.0 + '@smithy/types': 3.6.0 + bowser: 2.11.0 + tslib: 2.6.3 + + '@aws-sdk/util-user-agent-node@3.679.0': + dependencies: + '@aws-sdk/middleware-user-agent': 3.679.0 + '@aws-sdk/types': 3.679.0 + '@smithy/node-config-provider': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@aws-sdk/util-utf8-browser@3.259.0': + dependencies: + tslib: 2.6.3 + '@babel/code-frame@7.24.6': dependencies: '@babel/highlight': 7.24.6 @@ -5040,7 +5926,7 @@ snapshots: '@devil7softwares/pos@1.0.2': {} - '@dqbd/tiktoken@1.0.15': {} + '@dqbd/tiktoken@1.0.17': {} '@flydotio/dockerfile@0.4.11': dependencies: @@ -5338,13 +6224,13 @@ snapshots: '@js-sdsl/ordered-map@4.4.2': {} - '@langchain/core@0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8))': + '@langchain/core@0.2.12(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8))': dependencies: ansi-styles: 5.2.0 camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.12 - langsmith: 0.1.34(@langchain/core@0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)))(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) + langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu) ml-distance: 4.0.1 mustache: 4.2.0 p-queue: 6.6.2 @@ -5356,9 +6242,9 @@ snapshots: - langchain - openai - '@langchain/openai@0.2.1(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))': + '@langchain/openai@0.2.1(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))': dependencies: - '@langchain/core': 0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) + '@langchain/core': 0.2.12(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) js-tiktoken: 1.0.12 openai: 4.57.0(zod@3.23.8) zod: 3.23.8 @@ -5367,9 +6253,9 @@ snapshots: - encoding - langchain - '@langchain/textsplitters@0.0.3(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8))': + '@langchain/textsplitters@0.0.3(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8))': dependencies: - '@langchain/core': 0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) + '@langchain/core': 0.2.12(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) js-tiktoken: 1.0.12 transitivePeerDependencies: - langchain @@ -6495,6 +7381,340 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 + '@smithy/abort-controller@3.1.6': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/config-resolver@3.0.10': + dependencies: + '@smithy/node-config-provider': 3.1.9 + '@smithy/types': 3.6.0 + '@smithy/util-config-provider': 3.0.0 + '@smithy/util-middleware': 3.0.8 + tslib: 2.6.3 + + '@smithy/core@2.5.1': + dependencies: + '@smithy/middleware-serde': 3.0.8 + '@smithy/protocol-http': 4.1.5 + '@smithy/types': 3.6.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-stream': 3.2.1 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + + '@smithy/credential-provider-imds@3.2.5': + dependencies: + '@smithy/node-config-provider': 3.1.9 + '@smithy/property-provider': 3.1.8 + '@smithy/types': 3.6.0 + '@smithy/url-parser': 3.0.8 + tslib: 2.6.3 + + '@smithy/eventstream-codec@1.1.0': + dependencies: + '@aws-crypto/crc32': 3.0.0 + '@smithy/types': 1.2.0 + '@smithy/util-hex-encoding': 1.1.0 + tslib: 2.6.3 + + '@smithy/fetch-http-handler@3.2.9': + dependencies: + '@smithy/protocol-http': 4.1.5 + '@smithy/querystring-builder': 3.0.8 + '@smithy/types': 3.6.0 + '@smithy/util-base64': 3.0.0 + tslib: 2.6.3 + + '@smithy/fetch-http-handler@4.0.0': + dependencies: + '@smithy/protocol-http': 4.1.5 + '@smithy/querystring-builder': 3.0.8 + '@smithy/types': 3.6.0 + '@smithy/util-base64': 3.0.0 + tslib: 2.6.3 + + '@smithy/hash-node@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + '@smithy/util-buffer-from': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + + '@smithy/invalid-dependency@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/is-array-buffer@1.1.0': + dependencies: + tslib: 2.6.3 + + '@smithy/is-array-buffer@2.2.0': + dependencies: + tslib: 2.6.3 + + '@smithy/is-array-buffer@3.0.0': + dependencies: + tslib: 2.6.3 + + '@smithy/middleware-content-length@3.0.10': + dependencies: + '@smithy/protocol-http': 4.1.5 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/middleware-endpoint@3.2.1': + dependencies: + '@smithy/core': 2.5.1 + '@smithy/middleware-serde': 3.0.8 + '@smithy/node-config-provider': 3.1.9 + '@smithy/shared-ini-file-loader': 3.1.9 + '@smithy/types': 3.6.0 + '@smithy/url-parser': 3.0.8 + '@smithy/util-middleware': 3.0.8 + tslib: 2.6.3 + + '@smithy/middleware-retry@3.0.25': + dependencies: + '@smithy/node-config-provider': 3.1.9 + '@smithy/protocol-http': 4.1.5 + '@smithy/service-error-classification': 3.0.8 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-retry': 3.0.8 + tslib: 2.6.3 + uuid: 9.0.1 + + '@smithy/middleware-serde@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/middleware-stack@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/node-config-provider@3.1.9': + dependencies: + '@smithy/property-provider': 3.1.8 + '@smithy/shared-ini-file-loader': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/node-http-handler@3.2.5': + dependencies: + '@smithy/abort-controller': 3.1.6 + '@smithy/protocol-http': 4.1.5 + '@smithy/querystring-builder': 3.0.8 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/property-provider@3.1.8': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/protocol-http@1.2.0': + dependencies: + '@smithy/types': 1.2.0 + tslib: 2.6.3 + + '@smithy/protocol-http@4.1.5': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/querystring-builder@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + '@smithy/util-uri-escape': 3.0.0 + tslib: 2.6.3 + + '@smithy/querystring-parser@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/service-error-classification@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + + '@smithy/shared-ini-file-loader@3.1.9': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/signature-v4@1.1.0': + dependencies: + '@smithy/eventstream-codec': 1.1.0 + '@smithy/is-array-buffer': 1.1.0 + '@smithy/types': 1.2.0 + '@smithy/util-hex-encoding': 1.1.0 + '@smithy/util-middleware': 1.1.0 + '@smithy/util-uri-escape': 1.1.0 + '@smithy/util-utf8': 1.1.0 + tslib: 2.6.3 + + '@smithy/signature-v4@4.2.1': + dependencies: + '@smithy/is-array-buffer': 3.0.0 + '@smithy/protocol-http': 4.1.5 + '@smithy/types': 3.6.0 + '@smithy/util-hex-encoding': 3.0.0 + '@smithy/util-middleware': 3.0.8 + '@smithy/util-uri-escape': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + + '@smithy/smithy-client@3.4.2': + dependencies: + '@smithy/core': 2.5.1 + '@smithy/middleware-endpoint': 3.2.1 + '@smithy/middleware-stack': 3.0.8 + '@smithy/protocol-http': 4.1.5 + '@smithy/types': 3.6.0 + '@smithy/util-stream': 3.2.1 + tslib: 2.6.3 + + '@smithy/types@1.2.0': + dependencies: + tslib: 2.6.3 + + '@smithy/types@3.6.0': + dependencies: + tslib: 2.6.3 + + '@smithy/url-parser@3.0.8': + dependencies: + '@smithy/querystring-parser': 3.0.8 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/util-base64@3.0.0': + dependencies: + '@smithy/util-buffer-from': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + + '@smithy/util-body-length-browser@3.0.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-body-length-node@3.0.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-buffer-from@1.1.0': + dependencies: + '@smithy/is-array-buffer': 1.1.0 + tslib: 2.6.3 + + '@smithy/util-buffer-from@2.2.0': + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.6.3 + + '@smithy/util-buffer-from@3.0.0': + dependencies: + '@smithy/is-array-buffer': 3.0.0 + tslib: 2.6.3 + + '@smithy/util-config-provider@3.0.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-defaults-mode-browser@3.0.25': + dependencies: + '@smithy/property-provider': 3.1.8 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + bowser: 2.11.0 + tslib: 2.6.3 + + '@smithy/util-defaults-mode-node@3.0.25': + dependencies: + '@smithy/config-resolver': 3.0.10 + '@smithy/credential-provider-imds': 3.2.5 + '@smithy/node-config-provider': 3.1.9 + '@smithy/property-provider': 3.1.8 + '@smithy/smithy-client': 3.4.2 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/util-endpoints@2.1.4': + dependencies: + '@smithy/node-config-provider': 3.1.9 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/util-hex-encoding@1.1.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-hex-encoding@3.0.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-middleware@1.1.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-middleware@3.0.8': + dependencies: + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/util-retry@3.0.8': + dependencies: + '@smithy/service-error-classification': 3.0.8 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + + '@smithy/util-stream@3.2.1': + dependencies: + '@smithy/fetch-http-handler': 4.0.0 + '@smithy/node-http-handler': 3.2.5 + '@smithy/types': 3.6.0 + '@smithy/util-base64': 3.0.0 + '@smithy/util-buffer-from': 3.0.0 + '@smithy/util-hex-encoding': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + + '@smithy/util-uri-escape@1.1.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-uri-escape@3.0.0': + dependencies: + tslib: 2.6.3 + + '@smithy/util-utf8@1.1.0': + dependencies: + '@smithy/util-buffer-from': 1.1.0 + tslib: 2.6.3 + + '@smithy/util-utf8@2.3.0': + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.6.3 + + '@smithy/util-utf8@3.0.0': + dependencies: + '@smithy/util-buffer-from': 3.0.0 + tslib: 2.6.3 + + '@smithy/util-waiter@3.1.7': + dependencies: + '@smithy/abort-controller': 3.1.6 + '@smithy/types': 3.6.0 + tslib: 2.6.3 + '@stdlib/assert-has-own-property@0.0.7': {} '@stdlib/assert-has-symbol-support@0.0.8': @@ -7106,6 +8326,8 @@ snapshots: bottleneck@2.19.5: {} + bowser@2.11.0: {} + brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 @@ -7263,6 +8485,25 @@ snapshots: co@4.6.0: {} + cohere-ai@7.14.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)): + dependencies: + '@aws-sdk/client-sagemaker': 3.679.0 + '@aws-sdk/credential-providers': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)) + '@aws-sdk/protocol-http': 3.374.0 + '@aws-sdk/signature-v4': 3.374.0 + form-data: 4.0.0 + form-data-encoder: 4.0.2 + formdata-node: 6.0.3 + js-base64: 3.7.2 + node-fetch: 2.7.0 + qs: 6.11.2 + readable-stream: 4.5.2 + url-join: 4.0.1 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + - encoding + cohere@1.1.1: {} collect-v8-coverage@1.0.2: {} @@ -7650,6 +8891,10 @@ snapshots: fast-safe-stringify@2.1.1: {} + fast-xml-parser@4.4.1: + dependencies: + strnum: 1.0.5 + fb-watchman@2.0.2: dependencies: bser: 2.1.1 @@ -7701,6 +8946,8 @@ snapshots: form-data-encoder@1.7.2: {} + form-data-encoder@4.0.2: {} + form-data@4.0.0: dependencies: asynckit: 0.4.0 @@ -7712,6 +8959,8 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 4.0.0-beta.3 + formdata-node@6.0.3: {} + formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -8447,6 +9696,8 @@ snapshots: joplin-turndown-plugin-gfm@1.0.12: {} + js-base64@3.7.2: {} + js-beautify@1.15.1: dependencies: config-chain: 1.1.13 @@ -8513,17 +9764,17 @@ snapshots: koffi@2.9.0: {} - langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0): + langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0): dependencies: - '@langchain/core': 0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) - '@langchain/openai': 0.2.1(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0)) - '@langchain/textsplitters': 0.0.3(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) + '@langchain/core': 0.2.12(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) + '@langchain/openai': 0.2.1(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0)) + '@langchain/textsplitters': 0.0.3(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) binary-extensions: 2.3.0 js-tiktoken: 1.0.12 js-yaml: 4.1.0 jsonpointer: 5.0.1 langchainhub: 0.0.11 - langsmith: 0.1.34(@langchain/core@0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)))(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) + langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu) ml-distance: 4.0.1 openapi-types: 12.1.3 p-retry: 4.6.2 @@ -8532,14 +9783,16 @@ snapshots: zod: 3.23.8 zod-to-json-schema: 3.23.1(zod@3.23.8) optionalDependencies: + '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0) '@supabase/supabase-js': 2.44.2 axios: 1.7.2 cheerio: 1.0.0-rc.12 + fast-xml-parser: 4.4.1 handlebars: 4.7.8 html-to-text: 9.0.5 ioredis: 5.4.1 mammoth: 1.7.2 - mongodb: 6.6.2(socks@2.8.3) + mongodb: 6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3) pdf-parse: 1.1.1 puppeteer: 22.12.1(typescript@5.4.5) redis: 4.6.14 @@ -8550,7 +9803,7 @@ snapshots: langchainhub@0.0.11: {} - langsmith@0.1.34(@langchain/core@0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)))(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)): + langsmith@0.1.34(zyeavx4tfqw3smbbpiinhfxxeu): dependencies: '@types/uuid': 9.0.8 commander: 10.0.1 @@ -8559,8 +9812,8 @@ snapshots: p-retry: 4.6.2 uuid: 9.0.1 optionalDependencies: - '@langchain/core': 0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) - langchain: 0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0) + '@langchain/core': 0.2.12(langchain@0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) + langchain: 0.2.8(@aws-sdk/credential-provider-node@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0))(@aws-sdk/client-sts@3.679.0))(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(fast-xml-parser@4.4.1)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0) openai: 4.57.0(zod@3.23.8) languagedetect@2.0.0: {} @@ -8768,19 +10021,20 @@ snapshots: '@types/whatwg-url': 11.0.5 whatwg-url: 13.0.0 - mongodb@6.6.2(socks@2.8.3): + mongodb@6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3): dependencies: '@mongodb-js/saslprep': 1.1.7 bson: 6.8.0 mongodb-connection-string-url: 3.0.1 optionalDependencies: + '@aws-sdk/credential-providers': 3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)) socks: 2.8.3 - mongoose@8.4.4(socks@2.8.3): + mongoose@8.4.4(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3): dependencies: bson: 6.8.0 kareem: 2.6.3 - mongodb: 6.6.2(socks@2.8.3) + mongodb: 6.6.2(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3) mpath: 0.9.0 mquery: 5.0.0 ms: 2.1.3 @@ -8829,7 +10083,7 @@ snapshots: natural-compare@1.4.0: {} - natural@7.0.7(socks@2.8.3): + natural@7.0.7(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3): dependencies: afinn-165: 1.0.4 afinn-165-financialmarketnews: 3.0.0 @@ -8837,7 +10091,7 @@ snapshots: dotenv: 16.4.5 http-server: 14.1.1 memjs: 1.3.2 - mongoose: 8.4.4(socks@2.8.3) + mongoose: 8.4.4(@aws-sdk/credential-providers@3.679.0(@aws-sdk/client-sso-oidc@3.679.0(@aws-sdk/client-sts@3.679.0)))(socks@2.8.3) pg: 8.12.0 redis: 4.6.14 safe-stable-stringify: 2.4.3 @@ -9208,7 +10462,7 @@ snapshots: csv-parse: 5.5.6 gpt3-tokenizer: 1.1.5 openai: 3.3.0 - typescript: 5.6.2 + typescript: 5.6.3 uuid: 9.0.1 zod: 3.23.8 transitivePeerDependencies: @@ -9295,6 +10549,10 @@ snapshots: dependencies: side-channel: 1.0.6 + qs@6.11.2: + dependencies: + side-channel: 1.0.6 + qs@6.12.2: dependencies: side-channel: 1.0.6 @@ -9643,6 +10901,8 @@ snapshots: '@types/node': 20.14.1 qs: 6.12.2 + strnum@1.0.5: {} + supabase@1.172.2: dependencies: bin-links: 4.0.4 @@ -9806,7 +11066,7 @@ snapshots: typescript@5.4.5: {} - typescript@5.6.2: {} + typescript@5.6.3: {} typesense@1.8.2(@babel/runtime@7.24.6): dependencies: diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts new file mode 100644 index 00000000..f4e93e7f --- /dev/null +++ b/apps/api/src/controllers/v1/extract.ts @@ -0,0 +1,197 @@ +import { Request, Response } from "express"; +import { Logger } from "../../lib/logger"; +import { + Document, + legacyDocumentConverter, + legacyExtractorOptions, + legacyScrapeOptions, + RequestWithAuth, + ExtractRequest, + extractRequestSchema, + ExtractResponse, + legacyCrawlerOptions, + MapDocument, +} from "./types"; +import { billTeam } from "../../services/billing/credit_billing"; +import { v4 as uuidv4 } from "uuid"; +import { numTokensFromString } from "../../lib/LLM-extraction/helpers"; +import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; +import { logJob } from "../../services/logging/log_job"; +import { getJobPriority } from "../../lib/job-priority"; +import { PlanType } from "../../types"; +import { getMapResults } from "./map"; +import { rerankDocuments } from "../../lib/extract/reranker"; +import { generateBasicCompletion } from "../../lib/extract/completions"; + + + +export async function extractController( + req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>, + res: Response +) { + req.body = extractRequestSchema.parse(req.body); + let earlyReturn = false; + + const origin = req.body.origin; + const timeout = req.body.timeout; +// const pageOptions = legacyScrapeOptions(req.body); +// const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined; + const jobId = uuidv4(); + + const startTime = new Date().getTime(); + const jobPriority = await getJobPriority({ + plan: req.auth.plan as PlanType, + team_id: req.auth.team_id, + basePriority: 10, + }); + + const urls = req.body.urls; + const mappedDocuments: MapDocument[] = []; + + const prompt = req.body.prompt; + const keywords = await generateBasicCompletion(`If the user's prompt is: "${prompt}", what are the most important keywords besides the extraction task? Output only the keywords, separated by commas.`); + + for (const url of urls) { + if (url.endsWith("/*")) { + const mapResults = await getMapResults({ + url: url.slice(0, -2), + search: req.body.prompt, + limit: 100, + ignoreSitemap: true, + includeSubdomains: false, + crawlerOptions: {}, + teamId: req.auth.team_id, + plan: req.auth.plan, + origin: req.body.origin, + subId: req.acuc?.sub_id, + includeMetadata: true + }); + // top 3 links + const top3Links = (mapResults.links as MapDocument[]).slice(0, 3); + console.log(top3Links); + // console.log(top3Links); + mappedDocuments.push(...(mapResults.links as MapDocument[])); + // transform mappedUrls to just documents + // we quickly rerank + const rerank = await rerankDocuments(mappedDocuments.map(x => `URL: ${x.url}\nTITLE: ${x.title}\nDESCRIPTION: ${x.description}`), "What URLs are most relevant to the following prompt: " + req.body.prompt.toLocaleLowerCase().replace("extract", " ").replace("extract ", " ")); + console.log(rerank); + } else { + mappedDocuments.push({ url }); + } + } + + req.body.urls = mappedDocuments.map(x => x.url); + + + +// const job = await addScrapeJob( +// { +// url: req.body.url, +// mode: "single_urls", +// crawlerOptions: {}, +// team_id: req.auth.team_id, +// plan: req.auth.plan, +// pageOptions, +// extractorOptions, +// origin: req.body.origin, +// is_scrape: true, +// }, +// {}, +// jobId, +// jobPriority +// ); + +// const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds : 0) + a, 0); + +// let doc: any | undefined; +// try { +// doc = (await waitForJob(job.id, timeout + totalWait))[0]; +// } catch (e) { +// Logger.error(`Error in scrapeController: ${e}`); +// if (e instanceof Error && e.message.startsWith("Job wait")) { +// return res.status(408).json({ +// success: false, +// error: "Request timed out", +// }); +// } else { +// return res.status(500).json({ +// success: false, +// error: `(Internal server error) - ${e && e?.message ? e.message : e} ${ +// extractorOptions && extractorOptions.mode !== "markdown" +// ? " - Could be due to LLM parsing issues" +// : "" +// }`, +// }); +// } +// } + +// await job.remove(); + +// if (!doc) { +// console.error("!!! PANIC DOC IS", doc, job); +// return res.status(200).json({ +// success: true, +// warning: "No page found", +// data: doc, +// }); +// } + +// delete doc.index; +// delete doc.provider; + +// const endTime = new Date().getTime(); +// const timeTakenInSeconds = (endTime - startTime) / 1000; +// const numTokens = +// doc && doc.markdown +// ? numTokensFromString(doc.markdown, "gpt-3.5-turbo") +// : 0; + +// let creditsToBeBilled = 1; // Assuming 1 credit per document +// if (earlyReturn) { +// // Don't bill if we're early returning +// return; +// } +// if(req.body.extract && req.body.formats.includes("extract")) { +// creditsToBeBilled = 5; +// } + +// billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(error => { +// Logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`); +// // Optionally, you could notify an admin or add to a retry queue here +// }); + +// if (!pageOptions || !pageOptions.includeRawHtml) { +// if (doc && doc.rawHtml) { +// delete doc.rawHtml; +// } +// } + +// if(pageOptions && pageOptions.includeExtract) { +// if(!pageOptions.includeMarkdown && doc && doc.markdown) { +// delete doc.markdown; +// } +// } + +// logJob({ +// job_id: jobId, +// success: true, +// message: "Scrape completed", +// num_docs: 1, +// docs: [doc], +// time_taken: timeTakenInSeconds, +// team_id: req.auth.team_id, +// mode: "scrape", +// url: req.body.url, +// crawlerOptions: {}, +// pageOptions: pageOptions, +// origin: origin, +// extractor_options: extractorOptions, +// num_tokens: numTokens, +// }); + + return res.status(200).json({ + success: true, + data: null, + scrape_id: origin?.includes("website") ? jobId : undefined, + }); +} diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index 5ed3dd51..7e74b43e 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -15,11 +15,11 @@ import { removeDuplicateUrls, } from "../../lib/validateUrl"; import { fireEngineMap } from "../../search/fireEngine"; -import { billTeam } from "../../services/billing/credit_billing"; -import { logJob } from "../../services/logging/log_job"; import { performCosineSimilarity } from "../../lib/map-cosine"; import { Logger } from "../../lib/logger"; import Redis from "ioredis"; +import { billTeam } from "../../services/billing/credit_billing"; +import { logJob } from "../../services/logging/log_job"; configDotenv(); const redis = new Redis(process.env.REDIS_URL); @@ -29,35 +29,50 @@ const MAX_MAP_LIMIT = 5000; // Max Links that "Smart /map" can return const MAX_FIRE_ENGINE_RESULTS = 1000; -export async function mapController( - req: RequestWithAuth<{}, MapResponse, MapRequest>, - res: Response -) { +interface MapOptions { + url: string; + search?: string; + limit?: number; + ignoreSitemap?: boolean; + includeSubdomains?: boolean; + crawlerOptions?: any; + teamId: string; + plan: string; + origin?: string; + subId?: string; + includeMetadata?: boolean; +} + +export async function getMapResults({ + url, + search, + limit = MAX_MAP_LIMIT, + ignoreSitemap = false, + includeSubdomains = false, + crawlerOptions = {}, + teamId, + plan, + origin, + subId, + includeMetadata = false, +}: MapOptions) { const startTime = new Date().getTime(); - - req.body = mapRequestSchema.parse(req.body); - - const limit: number = req.body.limit ?? MAX_MAP_LIMIT; - const id = uuidv4(); - let links: string[] = [req.body.url]; + let links: { url: string; title?: string; description?: string }[] = [{ url }]; const sc: StoredCrawl = { - originUrl: req.body.url, - crawlerOptions: legacyCrawlerOptions(req.body), + originUrl: url, + crawlerOptions, pageOptions: {}, - team_id: req.auth.team_id, + team_id: teamId, createdAt: Date.now(), - plan: req.auth.plan, + plan, }; const crawler = crawlToCrawler(id, sc); - let urlWithoutWww = req.body.url.replace("www.", ""); - - let mapUrl = req.body.search - ? `"${req.body.search}" site:${urlWithoutWww}` - : `site:${req.body.url}`; + let urlWithoutWww = url.replace("www.", ""); + let mapUrl = search ? `"${search}" site:${urlWithoutWww}` : `site:${url}`; const resultsPerPage = 100; const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage); @@ -81,12 +96,11 @@ export async function mapController( pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1)); allResults = await Promise.all(pagePromises); - await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours + await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); } - // Parallelize sitemap fetch with serper search const [sitemap, ...searchResults] = await Promise.all([ - req.body.ignoreSitemap ? null : crawler.tryGetSitemap(), + ignoreSitemap ? null : crawler.tryGetSitemap(), ...(cachedResult ? [] : pagePromises), ]); @@ -96,7 +110,7 @@ export async function mapController( if (sitemap !== null) { sitemap.forEach((x) => { - links.push(x.url); + links.push({ url: x.url }); }); } @@ -110,67 +124,96 @@ export async function mapController( } if (mapResults.length > 0) { - if (req.body.search) { - // Ensure all map results are first, maintaining their order + if (search) { links = [ - mapResults[0].url, - ...mapResults.slice(1).map((x) => x.url), + { url: mapResults[0].url, title: mapResults[0].title, description: mapResults[0].description }, + ...mapResults.slice(1).map((x) => ({ + url: x.url, + title: x.title, + description: x.description + })), ...links, ]; } else { - mapResults.map((x) => { - links.push(x.url); + mapResults.forEach((x) => { + links.push({ + url: x.url, + title: x.title, + description: x.description + }); }); } } - // Perform cosine similarity between the search query and the list of links - if (req.body.search) { - const searchQuery = req.body.search.toLowerCase(); - - links = performCosineSimilarity(links, searchQuery); + if (search) { + const filteredLinks = performCosineSimilarity(links.map(l => l.url), search.toLowerCase()); + links = links.filter(l => filteredLinks.includes(l.url)); } links = links .map((x) => { try { - return checkAndUpdateURLForMap(x).url.trim(); + return { ...x, url: checkAndUpdateURLForMap(x.url).url.trim() }; } catch (_) { return null; } }) .filter((x) => x !== null); - // allows for subdomains to be included - links = links.filter((x) => isSameDomain(x, req.body.url)); + links = links.filter((x) => isSameDomain(x.url, url)); - // if includeSubdomains is false, filter out subdomains - if (!req.body.includeSubdomains) { - links = links.filter((x) => isSameSubdomain(x, req.body.url)); + if (!includeSubdomains) { + links = links.filter((x) => isSameSubdomain(x.url, url)); } - // remove duplicates that could be due to http/https or www - links = removeDuplicateUrls(links); - - billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => { - Logger.error( - `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}` - ); - // Optionally, you could notify an admin or add to a retry queue here - }); + links = removeDuplicateUrls(links.map(l => l.url)).map(url => links.find(l => l.url === url)); const endTime = new Date().getTime(); const timeTakenInSeconds = (endTime - startTime) / 1000; const linksToReturn = links.slice(0, limit); - logJob({ - job_id: id, - success: links.length > 0, + return { + links: includeMetadata ? linksToReturn : linksToReturn.map(l => l.url), + scrapeId: origin?.includes("website") ? id : undefined, + timeTakenInSeconds, + id, + linksLength: links.length, + linksToReturnLength: linksToReturn.length, + docs: linksToReturn.map(l => l.url), + }; +} + +export async function mapController( + req: RequestWithAuth<{}, MapResponse, MapRequest>, + res: Response +) { + req.body = mapRequestSchema.parse(req.body); + + const results = await getMapResults({ + url: req.body.url, + search: req.body.search, + limit: req.body.limit, + ignoreSitemap: req.body.ignoreSitemap, + includeSubdomains: req.body.includeSubdomains, + crawlerOptions: legacyCrawlerOptions(req.body), + teamId: req.auth.team_id, + plan: req.auth.plan, + origin: req.body.origin, + subId: req.acuc?.sub_id, + }); + + await billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => { + Logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`); + }); + + await logJob({ + job_id: results.id, + success: results.linksLength > 0, message: "Map completed", - num_docs: linksToReturn.length, - docs: linksToReturn, - time_taken: timeTakenInSeconds, + num_docs: results.linksToReturnLength, + docs: results.docs, + time_taken: results.timeTakenInSeconds, team_id: req.auth.team_id, mode: "map", url: req.body.url, @@ -183,55 +226,7 @@ export async function mapController( return res.status(200).json({ success: true, - links: linksToReturn, - scrape_id: req.body.origin?.includes("website") ? id : undefined, + links: results.links.map(l => l.url), + scrape_id: results.scrapeId, }); } - -// Subdomain sitemap url checking - -// // For each result, check for subdomains, get their sitemaps and add them to the links -// const processedUrls = new Set(); -// const processedSubdomains = new Set(); - -// for (const result of links) { -// let url; -// let hostParts; -// try { -// url = new URL(result); -// hostParts = url.hostname.split('.'); -// } catch (e) { -// continue; -// } - -// console.log("hostParts", hostParts); -// // Check if it's a subdomain (more than 2 parts, and not 'www') -// if (hostParts.length > 2 && hostParts[0] !== 'www') { -// const subdomain = hostParts[0]; -// console.log("subdomain", subdomain); -// const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`; -// console.log("subdomainUrl", subdomainUrl); - -// if (!processedSubdomains.has(subdomainUrl)) { -// processedSubdomains.add(subdomainUrl); - -// const subdomainCrawl = crawlToCrawler(id, { -// originUrl: subdomainUrl, -// crawlerOptions: legacyCrawlerOptions(req.body), -// pageOptions: {}, -// team_id: req.auth.team_id, -// createdAt: Date.now(), -// plan: req.auth.plan, -// }); -// const subdomainSitemap = await subdomainCrawl.tryGetSitemap(); -// if (subdomainSitemap) { -// subdomainSitemap.forEach((x) => { -// if (!processedUrls.has(x.url)) { -// processedUrls.add(x.url); -// links.push(x.url); -// } -// }); -// } -// } -// } -// } diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 22ac6294..4e31dd86 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -121,8 +121,21 @@ export const scrapeOptions = z.object({ }).strict(strictMessage) + export type ScrapeOptions = z.infer; +export const extractV1Options = z.object({ + urls: url.array(), + prompt: z.string().optional(), + schema: z.any().optional(), + origin: z.string().optional().default("api"), + timeout: z.number().int().positive().finite().safe().default(60000), +}).strict(strictMessage) + +export type ExtractV1Options = z.infer; +export const extractRequestSchema = extractV1Options; +export type ExtractRequest = z.infer; + export const scrapeRequestSchema = scrapeOptions.extend({ url, origin: z.string().optional().default("api"), @@ -142,6 +155,8 @@ export const scrapeRequestSchema = scrapeOptions.extend({ return obj; }); + + export type ScrapeRequest = z.infer; export const batchScrapeRequestSchema = scrapeOptions.extend({ @@ -296,6 +311,21 @@ export interface ScrapeResponseRequestTest { error?: string; } +export type ExtractResponse = + | ErrorResponse + | { + success: true; + warning?: string; + data: Document; + scrape_id?: string; + }; + +export interface ExtractResponseRequestTest { + statusCode: number; + body: ExtractResponse; + error?: string; +} + export type CrawlResponse = | ErrorResponse | { @@ -492,3 +522,11 @@ export function legacyDocumentConverter(doc: any): Document { }, }; } + + + +export interface MapDocument { + url: string; + title?: string; + description?: string; +} \ No newline at end of file diff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.ts new file mode 100644 index 00000000..230584b4 --- /dev/null +++ b/apps/api/src/lib/extract/completions.ts @@ -0,0 +1,119 @@ +import OpenAI from "openai"; +import { encoding_for_model } from "@dqbd/tiktoken"; +import { TiktokenModel } from "@dqbd/tiktoken"; +import { ExtractOptions } from "../../controllers/v1/types"; +import { Document } from "../entities"; +import { z } from "zod"; + +const maxTokens = 32000; +const modifier = 4; + +export class LLMRefusalError extends Error { + constructor(refusal: string) { + super("LLM refused to extract the website's content"); + this.name = "LLMRefusalError"; + } +} + +interface GenerateCompletionsParams { + systemPrompt?: string; + prompt?: string; + schema?: any; + pagesContent: string; +} + +export async function generateBasicCompletion(prompt: string) { + const openai = new OpenAI(); + const model: TiktokenModel = + (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini"; + + const completion = await openai.chat.completions.create({ + model, + messages: [{ role: "user", content: prompt }], + }); + + return completion.choices[0].message.content; +} + +export async function generateFinalExtraction({ + pagesContent, + systemPrompt, + prompt, + schema, +}: GenerateCompletionsParams): Promise<{ + content: string; + metadata: { numTokens: number; warning: string }; +}> { + const openai = new OpenAI(); + const model: TiktokenModel = + (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini"; + + let extractionContent = pagesContent; + let numTokens = 0; + let warning = ""; + + const encoder = encoding_for_model(model); + try { + const tokens = encoder.encode(extractionContent); + numTokens = tokens.length; + } catch (error) { + extractionContent = extractionContent.slice(0, maxTokens * modifier); + warning = `Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (${maxTokens}) we support.`; + } finally { + encoder.free(); + } + + if (numTokens > maxTokens) { + extractionContent = extractionContent.slice(0, maxTokens * modifier); + warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`; + } + + if (schema && (schema.type === "array" || schema._type === "ZodArray")) { + schema = { + type: "object", + properties: { + items: schema, + }, + required: ["items"], + additionalProperties: false, + }; + } + + const jsonCompletion = await openai.beta.chat.completions.parse({ + model, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: [{ type: "text", text: extractionContent }] }, + { + role: "user", + content: prompt + ? `Transform the above content into structured JSON output based on the following user request: ${prompt}` + : "Transform the above content into structured JSON output.", + }, + ], + response_format: schema + ? { + type: "json_schema", + json_schema: { + name: "websiteContent", + schema: schema.shape, + strict: true, + }, + } + : { type: "json_object" }, + }); + + if (jsonCompletion.choices[0].message.refusal !== null) { + throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal); + } + + const extraction = jsonCompletion.choices[0].message.parsed; + + return { + content: extraction, + metadata: { + numTokens, + warning, + }, + }; +} diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts new file mode 100644 index 00000000..30aca441 --- /dev/null +++ b/apps/api/src/lib/extract/reranker.ts @@ -0,0 +1,22 @@ +import { CohereClient } from "cohere-ai"; +import { MapDocument } from "../../controllers/v1/types"; +const cohere = new CohereClient({ + token: process.env.COHERE_API_KEY, +}); + +export async function rerankDocuments( + documents: (string | Record)[], + query: string, + topN = 3, + model = "rerank-english-v3.0" +) { + const rerank = await cohere.v2.rerank({ + documents, + query, + topN, + model, + returnDocuments: true, + }); + + return rerank.results.sort((a, b) => b.relevanceScore - a.relevanceScore).map(x => ({ document: x.document, index: x.index, relevanceScore: x.relevanceScore })); +}