-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/main'
- Loading branch information
Showing
41 changed files
with
1,432 additions
and
89 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
const code = Deno.readTextFileSync(import.meta.dirname + '/../../tools/download-page/tool.ts'); | ||
const metadata = JSON.parse(Deno.readTextFileSync(import.meta.dirname + '/../../tools/download-page/metadata.json')); | ||
|
||
type INPUTS = { | ||
url: string; | ||
}; | ||
const X_SHINKAI_TOOL_ID = `example-${Math.random().toString(36).substring(2, 15)}`; | ||
const X_SHINKAI_APP_ID = `run-${Math.random().toString(36).substring(2, 15)}`; | ||
|
||
const base_url = Deno.env.get('SHINKAI_NODE_ADDR') ?? 'http://localhost:9950'; | ||
const token = Deno.env.get('BEARER_TOKEN') ?? 'debug'; | ||
const llm_provider = Deno.env.get('INITIAL_AGENT_NAMES') ? (Deno.env.get('INITIAL_AGENT_NAMES')??'').split(',')[0] : 'llama3_1_8b'; | ||
async function runCommandTest(parameters: INPUTS) { | ||
const response = await fetch(base_url + '/v2/code_execution', { | ||
method: 'POST', | ||
headers: { | ||
'Authorization': 'Bearer ' + token, | ||
'x-shinkai-tool-id': X_SHINKAI_TOOL_ID, | ||
'x-shinkai-app-id': X_SHINKAI_APP_ID, | ||
'x-shinkai-llm-provider': llm_provider, | ||
'Content-Type': 'application/json; charset=utf-8', | ||
}, | ||
body: JSON.stringify({ | ||
code, | ||
tool_type: 'denodynamic', | ||
llm_provider, | ||
tools: metadata.tools || [], | ||
parameters | ||
}) | ||
}); | ||
|
||
const data = await response.json(); | ||
return data; | ||
} | ||
|
||
export async function downloadPageTest() { | ||
const expect = (data: any, message: string | undefined, errorStr: string | undefined) => { | ||
if (!data) throw Error('[Check failed] ' + (message ?? '') + ' ' + (errorStr ?? '')); | ||
else console.log('[Check passed] ' + (message ?? '')); | ||
} | ||
|
||
const parametersA: INPUTS = { | ||
url: 'https://shinkai.com' | ||
} | ||
const dataA= await runCommandTest(parametersA); | ||
expect(dataA.markdown.match(/open.source/i), 'Page should mention open source', `Missing text on ${dataA.markdown.substring(0, 100)}...`); | ||
} | ||
|
||
// downloadPageTest().then(console.log).catch(console.error); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
{"type":"Deno","content":[{"name":"Download Pages","homepage":null,"author":"@@official.shinkai","version":"1.0.0","js_code":"\nimport chromePaths from 'npm:chrome-paths@1.0.1';\nimport TurndownService from 'npm:turndown@7.2.0';\nimport { addExtra } from 'npm:puppeteer-extra@3.3.6';\nimport rebrowserPuppeteer from 'npm:rebrowser-puppeteer@23.10.1';\nimport StealthPlugin from 'npm:puppeteer-extra-plugin-stealth@2.11.2';\n\nimport { getHomePath } from './shinkai-local-support.ts';\n\ntype Configurations = {\n chromePath?: string;\n};\n\ntype Parameters = {\n url: string;\n incognito?: boolean;\n};\n\ntype Result = { markdown: string };\n\nexport type Run<C extends Record<string, any>, I extends Record<string, any>, R extends Record<string, any>> = (config: C, inputs: I) => Promise<R>;\n\nconst puppeteer = addExtra(rebrowserPuppeteer as any);\nconst pluginStealth = StealthPlugin();\npluginStealth.enabledEvasions.delete('chrome.loadTimes');\npluginStealth.enabledEvasions.delete('chrome.runtime');\npuppeteer.use(pluginStealth);\n\nexport const run: Run<Configurations, Parameters, Result> = async (\n configurations: Configurations,\n parameters: Parameters,\n): Promise<Result> => {\n const chromePath =\n configurations?.chromePath ||\n Deno.env.get('CHROME_PATH') ||\n chromePaths.chrome ||\n chromePaths.chromium;\n if (!chromePath) {\n throw new Error('Chrome path not found');\n }\n console.log({ chromePath })\n const browser = await puppeteer.launch({\n executablePath: chromePath,\n args: ['--disable-blink-features=AutomationControlled'],\n });\n\n const page = await browser.newPage();\n \n console.log(\"Navigating to website...\");\n await page.goto(parameters.url);\n \n console.log('Waiting for the page to load...');\n await page.waitForNetworkIdle();\n \n console.log('Extracting HTML content...');\n const html = await page.content();\n\n console.log('Closing browser...');\n await browser.close();\n\n console.log('Saving HTML to file...');\n Deno.writeTextFileSync(await getHomePath() + '/download-page.html', html);\n\n console.log('Converting HTML to Markdown...');\n const turndownService = new TurndownService();\n const markdown = turndownService.turndown(html);\n return Promise.resolve({ markdown });\n};\n","tools":[],"config":[{"BasicConfig":{"key_name":"chromePath","description":"The path to the Chrome executable","required":false,"type":null,"key_value":null}}],"description":"Downloads a URL and converts its HTML content to Markdown","keywords":["HTML to Markdown","web page downloader","content conversion","URL to Markdown"],"input_args":{"type":"object","properties":{"url":{"type":"string","description":"A URL of a web page to download"}},"required":["url"]},"output_arg":{"json":""},"activated":false,"embedding":[-0.022859422,0.34394914,-0.26828787,-0.2738084,-0.63775945,-0.07745881,-1.0741392,0.1560609,0.37143725,0.019180652,0.20704071,0.27364534,-0.0067869863,0.52876467,0.6414323,-0.33678514,0.050067496,-0.93635535,-1.4170387,-0.11435936,-0.23238567,1.0892224,0.2120105,0.5584683,0.12247756,-0.064977966,0.14893794,-0.5124398,-0.96074337,-2.0346339,0.73753834,0.45415106,-0.08883569,-0.7438674,0.48413786,-0.58480775,0.05941936,-0.5025951,-0.18222712,-0.3229596,0.09404269,-0.6394208,-0.13328093,-0.35516846,0.26403305,0.2549522,0.011935851,-0.10359208,0.7161654,0.2805506,-0.3459439,-0.7086845,-0.3280931,-0.16880323,-0.38078505,-0.05401938,-0.40572488,0.0827634,0.102821976,0.112446666,-0.2216977,0.31631678,-4.125646,0.27062064,0.5810081,0.39316595,-0.09006638,-0.13725893,-0.23155731,0.06299147,0.22843449,-0.21436238,-0.16343841,-0.28368253,-0.07376786,-0.7834173,0.62592435,-0.18950966,-0.071794435,-0.8110028,0.026290458,0.8741538,0.0618044,0.5859047,-0.556113,0.3412014,0.046755597,-0.41176373,0.313688,-0.0122999,0.368527,-0.90580493,-0.079653844,0.5979872,-0.4543758,0.042234086,-0.10474573,0.24580964,0.1175344,3.1755562,0.80056226,-0.023514325,0.79566926,-0.7410976,-0.0014581755,-0.31112045,-0.039167665,0.100143805,0.5764502,0.1579162,0.65378535,-0.5400134,-0.020097598,0.018258024,0.35735926,-0.64368653,-0.84609747,-0.22342974,0.7505621,0.82978624,-0.4969715,0.5932594,-0.00894372,-0.20878446,0.1974807,0.16078371,-0.15507145,0.3443752,0.18211538,0.038514256,0.48453322,-0.36101848,-0.5905768,0.3267346,0.702126,0.23804805,0.14942536,-0.78500104,0.34118044,-0.7625391,0.2017467,-1.0103437,0.11651488,0.5579052,0.3436056,0.52296245,0.08520118,0.64077246,-0.4686281,-0.30036545,0.123490244,0.44900888,-0.48335668,-0.42536774,0.74946654,-0.068822786,-0.40755105,-0.24519834,-0.1692692,0.52134687,-0.45200285,-0.60263264,0.27810538,0.7042316,0.18795288,-0.3493522,0.046734825,0.22792357,0.6453644,0.451598,0.4749537,-0.23558003,-0.091785856,-0.3761208,0.19023523,-0.013168387,-0.27135915,-0.5617338,-0.0695993,-0.65293926,0.3943035,0.48974362,-0.5824184,-0.4601611,-0.002162248,0.28777352,0.3323654,0.29203975,0.8401477,0.75948477,-0.2863765,1.5633736,-1.0324217,-0.43823823,-0.39899313,-0.14089474,0.17630441,0.6572716,0.3455184,0.090465,0.023112308,-0.39716557,0.4068057,-0.13822418,-0.35577852,-0.021364532,0.4544103,0.026246114,0.18069449,-0.44668064,0.2769035,-0.73281,1.1664879,0.2932155,0.4086936,-0.35218203,-0.25461483,0.409438,0.24547005,0.5120449,0.1091405,0.28187823,-0.19503851,-0.47988772,-0.4020756,0.26754946,0.1451346,0.028484786,-0.5201925,-0.12774691,0.6918011,0.4549158,0.03442505,1.0406747,1.0184767,0.19375554,-0.2703382,0.8375021,0.20192006,-0.63289624,0.19444388,-0.16417834,-0.13579649,-0.2540757,-0.23970604,-1.1852467,-0.13074327,0.33187962,0.46423656,1.7475896,0.77317226,0.50678706,0.6598569,0.04874261,0.2757128,0.064221054,-2.202051,-0.26725778,-0.23615283,1.212763,-0.17240548,0.4422012,0.12729892,0.022317126,-0.2723784,-0.37636,-0.43468815,-1.0617322,-0.35068285,0.31613588,-0.35614374,0.45097005,0.30068332,-0.043214343,0.01916102,0.5010773,0.62649405,-0.31871548,-0.7229701,-0.6690801,0.32861578,-0.17093404,0.5118524,0.00037259609,-0.11062399,-0.35906035,0.13406049,0.44416454,-0.068312965,0.27592757,0.051042564,-0.49447364,-0.063102454,0.8905604,1.6143283,-0.1856372,-0.05961505,-0.35624927,-0.5815494,-0.21133251,-0.23919454,0.3232051,-0.13867204,-0.22855097,-0.93279076,-0.29881087,0.47200906,-0.49601215,-0.15198481,0.017862234,-0.32360247,-0.07765387,0.09425101,-0.24398094,0.6091076,-0.5912474,-0.2593696,0.047237284,-0.20522922,-1.7102749,-0.5065819,0.004834657,0.13834465,-0.19261189,-0.6813854,0.93593514,-0.1833483,-0.16403382,-0.21518946,1.5840325,0.62321234,-0.6040731,-0.65227884,0.421814,0.659497,-0.17027326,-0.077558756,-0.46503827,-0.67175186,0.034123674,0.9178276,1.7945781,0.045439854,0.014577113,0.19752969,0.25004318,-0.85095567,-1.5589556,-0.16364035,-0.26558465,-0.16827886,0.29318962,-0.6095997,-0.1322887,0.52664423,0.69676834,-0.31061935,-0.06737543,-0.7940618,2.265823,0.15195686,0.057539836,-0.032563098,-0.12275851,-0.060634516,0.18676114,0.72584385,-0.5401763,-0.56609005,-0.29295185,0.26685232,-0.19534579,0.35730702,0.4106815,0.62266856,0.33912587,0.23005708,0.6000891,0.2057446,0.54149806,0.43303984,0.21790418,-0.47692624,0.19675124],"result":{"type":"object","properties":{"markdown":{"description":"The Markdown content converted from the web page","type":"string"}},"required":["markdown"]},"sql_tables":null,"sql_queries":null,"file_inbox":null,"oauth":null,"assets":null,"runner":"any","operating_system":["linux","macos","windows"],"tool_set":""},false]} | ||
{"type":"Deno","content":[{"name":"Download Pages","homepage":null,"author":"@@official.shinkai","version":"1.0.0","js_code":"\nimport chromePaths from 'npm:chrome-paths@1.0.1';\nimport TurndownService from 'npm:turndown@7.2.0';\nimport { addExtra } from 'npm:puppeteer-extra@3.3.6';\nimport rebrowserPuppeteer from 'npm:rebrowser-puppeteer@23.10.1';\nimport StealthPlugin from 'npm:puppeteer-extra-plugin-stealth@2.11.2';\n\nimport { getHomePath } from './shinkai-local-support.ts';\n\ntype Configurations = {\n chromePath?: string;\n};\n\ntype Parameters = {\n url: string;\n incognito?: boolean;\n};\n\ntype Result = { markdown: string };\n\nexport type Run<C extends Record<string, any>, I extends Record<string, any>, R extends Record<string, any>> = (config: C, inputs: I) => Promise<R>;\n\nconst puppeteer = addExtra(rebrowserPuppeteer as any);\nconst pluginStealth = StealthPlugin();\npluginStealth.enabledEvasions.delete('chrome.loadTimes');\npluginStealth.enabledEvasions.delete('chrome.runtime');\npuppeteer.use(pluginStealth);\n\nexport const run: Run<Configurations, Parameters, Result> = async (\n configurations: Configurations,\n parameters: Parameters,\n): Promise<Result> => {\n const chromePath =\n configurations?.chromePath ||\n Deno.env.get('CHROME_PATH') ||\n chromePaths.chrome ||\n chromePaths.chromium;\n if (!chromePath) {\n throw new Error('Chrome path not found');\n }\n console.log({ chromePath })\n const browser = await puppeteer.launch({\n executablePath: chromePath,\n args: ['--disable-blink-features=AutomationControlled'],\n });\n\n const page = await browser.newPage();\n \n console.log(\"Navigating to website...\");\n await page.goto(parameters.url);\n \n console.log('Waiting for the page to load...');\n await page.waitForNetworkIdle();\n \n console.log('Extracting HTML content...');\n const html = await page.content();\n\n console.log('Closing browser...');\n await browser.close();\n\n console.log('Saving HTML to file...');\n const safeFileName = parameters.url.replace(/\\./g, '_').replace(/[^a-zA-Z0-9_-]/g, '');\n const safeDate = new Date().toISOString().replace(/[:.]/g, '-');\n Deno.writeTextFileSync(await getHomePath() + `/raw.${safeFileName}.${safeDate}.html`, html);\n\n console.log('Converting HTML to Markdown...');\n const turndownService = new TurndownService();\n const markdown = turndownService\n .remove(['script', 'style', 'title'])\n .turndown(html);\n return Promise.resolve({ markdown });\n};\n","tools":[],"config":[{"BasicConfig":{"key_name":"chromePath","description":"The path to the Chrome executable","required":false,"type":null,"key_value":null}}],"description":"Downloads a URL and converts its HTML content to Markdown","keywords":["HTML to Markdown","web page downloader","content conversion","URL to Markdown"],"input_args":{"type":"object","properties":{"url":{"type":"string","description":"A URL of a web page to download"}},"required":["url"]},"output_arg":{"json":""},"activated":false,"embedding":[-0.022859422,0.34394914,-0.26828787,-0.2738084,-0.63775945,-0.07745881,-1.0741392,0.1560609,0.37143725,0.019180652,0.20704071,0.27364534,-0.0067869863,0.52876467,0.6414323,-0.33678514,0.050067496,-0.93635535,-1.4170387,-0.11435936,-0.23238567,1.0892224,0.2120105,0.5584683,0.12247756,-0.064977966,0.14893794,-0.5124398,-0.96074337,-2.0346339,0.73753834,0.45415106,-0.08883569,-0.7438674,0.48413786,-0.58480775,0.05941936,-0.5025951,-0.18222712,-0.3229596,0.09404269,-0.6394208,-0.13328093,-0.35516846,0.26403305,0.2549522,0.011935851,-0.10359208,0.7161654,0.2805506,-0.3459439,-0.7086845,-0.3280931,-0.16880323,-0.38078505,-0.05401938,-0.40572488,0.0827634,0.102821976,0.112446666,-0.2216977,0.31631678,-4.125646,0.27062064,0.5810081,0.39316595,-0.09006638,-0.13725893,-0.23155731,0.06299147,0.22843449,-0.21436238,-0.16343841,-0.28368253,-0.07376786,-0.7834173,0.62592435,-0.18950966,-0.071794435,-0.8110028,0.026290458,0.8741538,0.0618044,0.5859047,-0.556113,0.3412014,0.046755597,-0.41176373,0.313688,-0.0122999,0.368527,-0.90580493,-0.079653844,0.5979872,-0.4543758,0.042234086,-0.10474573,0.24580964,0.1175344,3.1755562,0.80056226,-0.023514325,0.79566926,-0.7410976,-0.0014581755,-0.31112045,-0.039167665,0.100143805,0.5764502,0.1579162,0.65378535,-0.5400134,-0.020097598,0.018258024,0.35735926,-0.64368653,-0.84609747,-0.22342974,0.7505621,0.82978624,-0.4969715,0.5932594,-0.00894372,-0.20878446,0.1974807,0.16078371,-0.15507145,0.3443752,0.18211538,0.038514256,0.48453322,-0.36101848,-0.5905768,0.3267346,0.702126,0.23804805,0.14942536,-0.78500104,0.34118044,-0.7625391,0.2017467,-1.0103437,0.11651488,0.5579052,0.3436056,0.52296245,0.08520118,0.64077246,-0.4686281,-0.30036545,0.123490244,0.44900888,-0.48335668,-0.42536774,0.74946654,-0.068822786,-0.40755105,-0.24519834,-0.1692692,0.52134687,-0.45200285,-0.60263264,0.27810538,0.7042316,0.18795288,-0.3493522,0.046734825,0.22792357,0.6453644,0.451598,0.4749537,-0.23558003,-0.091785856,-0.3761208,0.19023523,-0.013168387,-0.27135915,-0.5617338,-0.0695993,-0.65293926,0.3943035,0.48974362,-0.5824184,-0.4601611,-0.002162248,0.28777352,0.3323654,0.29203975,0.8401477,0.75948477,-0.2863765,1.5633736,-1.0324217,-0.43823823,-0.39899313,-0.14089474,0.17630441,0.6572716,0.3455184,0.090465,0.023112308,-0.39716557,0.4068057,-0.13822418,-0.35577852,-0.021364532,0.4544103,0.026246114,0.18069449,-0.44668064,0.2769035,-0.73281,1.1664879,0.2932155,0.4086936,-0.35218203,-0.25461483,0.409438,0.24547005,0.5120449,0.1091405,0.28187823,-0.19503851,-0.47988772,-0.4020756,0.26754946,0.1451346,0.028484786,-0.5201925,-0.12774691,0.6918011,0.4549158,0.03442505,1.0406747,1.0184767,0.19375554,-0.2703382,0.8375021,0.20192006,-0.63289624,0.19444388,-0.16417834,-0.13579649,-0.2540757,-0.23970604,-1.1852467,-0.13074327,0.33187962,0.46423656,1.7475896,0.77317226,0.50678706,0.6598569,0.04874261,0.2757128,0.064221054,-2.202051,-0.26725778,-0.23615283,1.212763,-0.17240548,0.4422012,0.12729892,0.022317126,-0.2723784,-0.37636,-0.43468815,-1.0617322,-0.35068285,0.31613588,-0.35614374,0.45097005,0.30068332,-0.043214343,0.01916102,0.5010773,0.62649405,-0.31871548,-0.7229701,-0.6690801,0.32861578,-0.17093404,0.5118524,0.00037259609,-0.11062399,-0.35906035,0.13406049,0.44416454,-0.068312965,0.27592757,0.051042564,-0.49447364,-0.063102454,0.8905604,1.6143283,-0.1856372,-0.05961505,-0.35624927,-0.5815494,-0.21133251,-0.23919454,0.3232051,-0.13867204,-0.22855097,-0.93279076,-0.29881087,0.47200906,-0.49601215,-0.15198481,0.017862234,-0.32360247,-0.07765387,0.09425101,-0.24398094,0.6091076,-0.5912474,-0.2593696,0.047237284,-0.20522922,-1.7102749,-0.5065819,0.004834657,0.13834465,-0.19261189,-0.6813854,0.93593514,-0.1833483,-0.16403382,-0.21518946,1.5840325,0.62321234,-0.6040731,-0.65227884,0.421814,0.659497,-0.17027326,-0.077558756,-0.46503827,-0.67175186,0.034123674,0.9178276,1.7945781,0.045439854,0.014577113,0.19752969,0.25004318,-0.85095567,-1.5589556,-0.16364035,-0.26558465,-0.16827886,0.29318962,-0.6095997,-0.1322887,0.52664423,0.69676834,-0.31061935,-0.06737543,-0.7940618,2.265823,0.15195686,0.057539836,-0.032563098,-0.12275851,-0.060634516,0.18676114,0.72584385,-0.5401763,-0.56609005,-0.29295185,0.26685232,-0.19534579,0.35730702,0.4106815,0.62266856,0.33912587,0.23005708,0.6000891,0.2057446,0.54149806,0.43303984,0.21790418,-0.47692624,0.19675124],"result":{"type":"object","properties":{"markdown":{"description":"The Markdown content converted from the web page","type":"string"}},"required":["markdown"]},"sql_tables":null,"sql_queries":null,"file_inbox":null,"oauth":null,"assets":null,"runner":"any","operating_system":["linux","macos","windows"],"tool_set":""},false]} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Oops, something went wrong.