diff --git a/scripts/tests/download-page.test.ts b/scripts/tests/download-page.test.ts new file mode 100644 index 00000000..294a10ab --- /dev/null +++ b/scripts/tests/download-page.test.ts @@ -0,0 +1,49 @@ +const code = Deno.readTextFileSync(import.meta.dirname + '/../../tools/download-page/tool.ts'); +const metadata = JSON.parse(Deno.readTextFileSync(import.meta.dirname + '/../../tools/download-page/metadata.json')); + +type INPUTS = { + url: string; +}; +const X_SHINKAI_TOOL_ID = `example-${Math.random().toString(36).substring(2, 15)}`; +const X_SHINKAI_APP_ID = `run-${Math.random().toString(36).substring(2, 15)}`; + +const base_url = Deno.env.get('SHINKAI_NODE_ADDR') ?? 'http://localhost:9950'; +const token = Deno.env.get('BEARER_TOKEN') ?? 'debug'; +const llm_provider = Deno.env.get('INITIAL_AGENT_NAMES') ? (Deno.env.get('INITIAL_AGENT_NAMES')??'').split(',')[0] : 'llama3_1_8b'; +async function runCommandTest(parameters: INPUTS) { + const response = await fetch(base_url + '/v2/code_execution', { + method: 'POST', + headers: { + 'Authorization': 'Bearer ' + token, + 'x-shinkai-tool-id': X_SHINKAI_TOOL_ID, + 'x-shinkai-app-id': X_SHINKAI_APP_ID, + 'x-shinkai-llm-provider': llm_provider, + 'Content-Type': 'application/json; charset=utf-8', + }, + body: JSON.stringify({ + code, + tool_type: 'denodynamic', + llm_provider, + tools: metadata.tools || [], + parameters + }) + }); + + const data = await response.json(); + return data; +} + +export async function downloadPageTest() { + const expect = (data: any, message: string | undefined, errorStr: string | undefined) => { + if (!data) throw Error('[Check failed] ' + (message ?? '') + ' ' + (errorStr ?? '')); + else console.log('[Check passed] ' + (message ?? '')); + } + + const parametersA: INPUTS = { + url: 'https://shinkai.com' + } + const dataA= await runCommandTest(parametersA); + expect(dataA.markdown.match(/open.source/i), 'Page should mention open source', `Missing text on ${dataA.markdown.substring(0, 100)}...`); +} + +// downloadPageTest().then(console.log).catch(console.error); \ No newline at end of file diff --git a/scripts/zip.test.ts b/scripts/zip.test.ts index 435cec78..030b830c 100644 --- a/scripts/zip.test.ts +++ b/scripts/zip.test.ts @@ -3,6 +3,10 @@ import { getMetadata, processToolsDirectory, saveToolsInNode } from "./build_too import { join } from "https://deno.land/std/path/mod.ts"; import { exists } from "https://deno.land/std/fs/mod.ts"; import { stripVersion, systemTools } from "./build_tools/system.ts"; +import { memoryTest } from "./tests/tool-memory.test.ts"; +import { downloadPageTest } from "./tests/download-page.test.ts"; + +const TOOL_TESTS = !!Deno.env.get("TOOL_TESTS"); Deno.test("Check if related tools exist", async () => { const tools_raw = await processToolsDirectory(); @@ -68,6 +72,11 @@ Deno.test("Compare shinkai-node generated ZIP __tool.json vs .tool-dump.test.jso }); -// Deno.test("Check if memory is working", async () => { -// await memoryTest(); -// }); +if (TOOL_TESTS) { + Deno.test("Check if memory is working", async () => { + await memoryTest(); + }); + Deno.test("Check if download-page is working", async () => { + await downloadPageTest(); + }); +} diff --git a/tools/download-page/.tool-dump.test.json b/tools/download-page/.tool-dump.test.json index dc6d5682..c0c3b47e 100644 --- a/tools/download-page/.tool-dump.test.json +++ b/tools/download-page/.tool-dump.test.json @@ -1 +1 @@ -{"type":"Deno","content":[{"name":"Download Pages","homepage":null,"author":"@@official.shinkai","version":"1.0.0","js_code":"\nimport chromePaths from 'npm:chrome-paths@1.0.1';\nimport TurndownService from 'npm:turndown@7.2.0';\nimport { addExtra } from 'npm:puppeteer-extra@3.3.6';\nimport rebrowserPuppeteer from 'npm:rebrowser-puppeteer@23.10.1';\nimport StealthPlugin from 'npm:puppeteer-extra-plugin-stealth@2.11.2';\n\nimport { getHomePath } from './shinkai-local-support.ts';\n\ntype Configurations = {\n chromePath?: string;\n};\n\ntype Parameters = {\n url: string;\n incognito?: boolean;\n};\n\ntype Result = { markdown: string };\n\nexport type Run, I extends Record, R extends Record> = (config: C, inputs: I) => Promise;\n\nconst puppeteer = addExtra(rebrowserPuppeteer as any);\nconst pluginStealth = StealthPlugin();\npluginStealth.enabledEvasions.delete('chrome.loadTimes');\npluginStealth.enabledEvasions.delete('chrome.runtime');\npuppeteer.use(pluginStealth);\n\nexport const run: Run = async (\n configurations: Configurations,\n parameters: Parameters,\n): Promise => {\n const chromePath =\n configurations?.chromePath ||\n Deno.env.get('CHROME_PATH') ||\n chromePaths.chrome ||\n chromePaths.chromium;\n if (!chromePath) {\n throw new Error('Chrome path not found');\n }\n console.log({ chromePath })\n const browser = await puppeteer.launch({\n executablePath: chromePath,\n args: ['--disable-blink-features=AutomationControlled'],\n });\n\n const page = await browser.newPage();\n \n console.log(\"Navigating to website...\");\n await page.goto(parameters.url);\n \n console.log('Waiting for the page to load...');\n await page.waitForNetworkIdle();\n \n console.log('Extracting HTML content...');\n const html = await page.content();\n\n console.log('Closing browser...');\n await browser.close();\n\n console.log('Saving HTML to file...');\n Deno.writeTextFileSync(await getHomePath() + '/download-page.html', html);\n\n console.log('Converting HTML to Markdown...');\n const turndownService = new TurndownService();\n const markdown = turndownService.turndown(html);\n return Promise.resolve({ markdown });\n};\n","tools":[],"config":[{"BasicConfig":{"key_name":"chromePath","description":"The path to the Chrome executable","required":false,"type":null,"key_value":null}}],"description":"Downloads a URL and converts its HTML content to Markdown","keywords":["HTML to Markdown","web page downloader","content conversion","URL to Markdown"],"input_args":{"type":"object","properties":{"url":{"type":"string","description":"A URL of a web page to download"}},"required":["url"]},"output_arg":{"json":""},"activated":false,"embedding":[-0.022859422,0.34394914,-0.26828787,-0.2738084,-0.63775945,-0.07745881,-1.0741392,0.1560609,0.37143725,0.019180652,0.20704071,0.27364534,-0.0067869863,0.52876467,0.6414323,-0.33678514,0.050067496,-0.93635535,-1.4170387,-0.11435936,-0.23238567,1.0892224,0.2120105,0.5584683,0.12247756,-0.064977966,0.14893794,-0.5124398,-0.96074337,-2.0346339,0.73753834,0.45415106,-0.08883569,-0.7438674,0.48413786,-0.58480775,0.05941936,-0.5025951,-0.18222712,-0.3229596,0.09404269,-0.6394208,-0.13328093,-0.35516846,0.26403305,0.2549522,0.011935851,-0.10359208,0.7161654,0.2805506,-0.3459439,-0.7086845,-0.3280931,-0.16880323,-0.38078505,-0.05401938,-0.40572488,0.0827634,0.102821976,0.112446666,-0.2216977,0.31631678,-4.125646,0.27062064,0.5810081,0.39316595,-0.09006638,-0.13725893,-0.23155731,0.06299147,0.22843449,-0.21436238,-0.16343841,-0.28368253,-0.07376786,-0.7834173,0.62592435,-0.18950966,-0.071794435,-0.8110028,0.026290458,0.8741538,0.0618044,0.5859047,-0.556113,0.3412014,0.046755597,-0.41176373,0.313688,-0.0122999,0.368527,-0.90580493,-0.079653844,0.5979872,-0.4543758,0.042234086,-0.10474573,0.24580964,0.1175344,3.1755562,0.80056226,-0.023514325,0.79566926,-0.7410976,-0.0014581755,-0.31112045,-0.039167665,0.100143805,0.5764502,0.1579162,0.65378535,-0.5400134,-0.020097598,0.018258024,0.35735926,-0.64368653,-0.84609747,-0.22342974,0.7505621,0.82978624,-0.4969715,0.5932594,-0.00894372,-0.20878446,0.1974807,0.16078371,-0.15507145,0.3443752,0.18211538,0.038514256,0.48453322,-0.36101848,-0.5905768,0.3267346,0.702126,0.23804805,0.14942536,-0.78500104,0.34118044,-0.7625391,0.2017467,-1.0103437,0.11651488,0.5579052,0.3436056,0.52296245,0.08520118,0.64077246,-0.4686281,-0.30036545,0.123490244,0.44900888,-0.48335668,-0.42536774,0.74946654,-0.068822786,-0.40755105,-0.24519834,-0.1692692,0.52134687,-0.45200285,-0.60263264,0.27810538,0.7042316,0.18795288,-0.3493522,0.046734825,0.22792357,0.6453644,0.451598,0.4749537,-0.23558003,-0.091785856,-0.3761208,0.19023523,-0.013168387,-0.27135915,-0.5617338,-0.0695993,-0.65293926,0.3943035,0.48974362,-0.5824184,-0.4601611,-0.002162248,0.28777352,0.3323654,0.29203975,0.8401477,0.75948477,-0.2863765,1.5633736,-1.0324217,-0.43823823,-0.39899313,-0.14089474,0.17630441,0.6572716,0.3455184,0.090465,0.023112308,-0.39716557,0.4068057,-0.13822418,-0.35577852,-0.021364532,0.4544103,0.026246114,0.18069449,-0.44668064,0.2769035,-0.73281,1.1664879,0.2932155,0.4086936,-0.35218203,-0.25461483,0.409438,0.24547005,0.5120449,0.1091405,0.28187823,-0.19503851,-0.47988772,-0.4020756,0.26754946,0.1451346,0.028484786,-0.5201925,-0.12774691,0.6918011,0.4549158,0.03442505,1.0406747,1.0184767,0.19375554,-0.2703382,0.8375021,0.20192006,-0.63289624,0.19444388,-0.16417834,-0.13579649,-0.2540757,-0.23970604,-1.1852467,-0.13074327,0.33187962,0.46423656,1.7475896,0.77317226,0.50678706,0.6598569,0.04874261,0.2757128,0.064221054,-2.202051,-0.26725778,-0.23615283,1.212763,-0.17240548,0.4422012,0.12729892,0.022317126,-0.2723784,-0.37636,-0.43468815,-1.0617322,-0.35068285,0.31613588,-0.35614374,0.45097005,0.30068332,-0.043214343,0.01916102,0.5010773,0.62649405,-0.31871548,-0.7229701,-0.6690801,0.32861578,-0.17093404,0.5118524,0.00037259609,-0.11062399,-0.35906035,0.13406049,0.44416454,-0.068312965,0.27592757,0.051042564,-0.49447364,-0.063102454,0.8905604,1.6143283,-0.1856372,-0.05961505,-0.35624927,-0.5815494,-0.21133251,-0.23919454,0.3232051,-0.13867204,-0.22855097,-0.93279076,-0.29881087,0.47200906,-0.49601215,-0.15198481,0.017862234,-0.32360247,-0.07765387,0.09425101,-0.24398094,0.6091076,-0.5912474,-0.2593696,0.047237284,-0.20522922,-1.7102749,-0.5065819,0.004834657,0.13834465,-0.19261189,-0.6813854,0.93593514,-0.1833483,-0.16403382,-0.21518946,1.5840325,0.62321234,-0.6040731,-0.65227884,0.421814,0.659497,-0.17027326,-0.077558756,-0.46503827,-0.67175186,0.034123674,0.9178276,1.7945781,0.045439854,0.014577113,0.19752969,0.25004318,-0.85095567,-1.5589556,-0.16364035,-0.26558465,-0.16827886,0.29318962,-0.6095997,-0.1322887,0.52664423,0.69676834,-0.31061935,-0.06737543,-0.7940618,2.265823,0.15195686,0.057539836,-0.032563098,-0.12275851,-0.060634516,0.18676114,0.72584385,-0.5401763,-0.56609005,-0.29295185,0.26685232,-0.19534579,0.35730702,0.4106815,0.62266856,0.33912587,0.23005708,0.6000891,0.2057446,0.54149806,0.43303984,0.21790418,-0.47692624,0.19675124],"result":{"type":"object","properties":{"markdown":{"description":"The Markdown content converted from the web page","type":"string"}},"required":["markdown"]},"sql_tables":null,"sql_queries":null,"file_inbox":null,"oauth":null,"assets":null,"runner":"any","operating_system":["linux","macos","windows"],"tool_set":""},false]} \ No newline at end of file +{"type":"Deno","content":[{"name":"Download Pages","homepage":null,"author":"@@official.shinkai","version":"1.0.0","js_code":"\nimport chromePaths from 'npm:chrome-paths@1.0.1';\nimport TurndownService from 'npm:turndown@7.2.0';\nimport { addExtra } from 'npm:puppeteer-extra@3.3.6';\nimport rebrowserPuppeteer from 'npm:rebrowser-puppeteer@23.10.1';\nimport StealthPlugin from 'npm:puppeteer-extra-plugin-stealth@2.11.2';\n\nimport { getHomePath } from './shinkai-local-support.ts';\n\ntype Configurations = {\n chromePath?: string;\n};\n\ntype Parameters = {\n url: string;\n incognito?: boolean;\n};\n\ntype Result = { markdown: string };\n\nexport type Run, I extends Record, R extends Record> = (config: C, inputs: I) => Promise;\n\nconst puppeteer = addExtra(rebrowserPuppeteer as any);\nconst pluginStealth = StealthPlugin();\npluginStealth.enabledEvasions.delete('chrome.loadTimes');\npluginStealth.enabledEvasions.delete('chrome.runtime');\npuppeteer.use(pluginStealth);\n\nexport const run: Run = async (\n configurations: Configurations,\n parameters: Parameters,\n): Promise => {\n const chromePath =\n configurations?.chromePath ||\n Deno.env.get('CHROME_PATH') ||\n chromePaths.chrome ||\n chromePaths.chromium;\n if (!chromePath) {\n throw new Error('Chrome path not found');\n }\n console.log({ chromePath })\n const browser = await puppeteer.launch({\n executablePath: chromePath,\n args: ['--disable-blink-features=AutomationControlled'],\n });\n\n const page = await browser.newPage();\n \n console.log(\"Navigating to website...\");\n await page.goto(parameters.url);\n \n console.log('Waiting for the page to load...');\n await page.waitForNetworkIdle();\n \n console.log('Extracting HTML content...');\n const html = await page.content();\n\n console.log('Closing browser...');\n await browser.close();\n\n console.log('Saving HTML to file...');\n const safeFileName = parameters.url.replace(/\\./g, '_').replace(/[^a-zA-Z0-9_-]/g, '');\n const safeDate = new Date().toISOString().replace(/[:.]/g, '-');\n Deno.writeTextFileSync(await getHomePath() + `/raw.${safeFileName}.${safeDate}.html`, html);\n\n console.log('Converting HTML to Markdown...');\n const turndownService = new TurndownService();\n const markdown = turndownService\n .remove(['script', 'style', 'title'])\n .turndown(html);\n return Promise.resolve({ markdown });\n};\n","tools":[],"config":[{"BasicConfig":{"key_name":"chromePath","description":"The path to the Chrome executable","required":false,"type":null,"key_value":null}}],"description":"Downloads a URL and converts its HTML content to Markdown","keywords":["HTML to Markdown","web page downloader","content conversion","URL to Markdown"],"input_args":{"type":"object","properties":{"url":{"type":"string","description":"A URL of a web page to download"}},"required":["url"]},"output_arg":{"json":""},"activated":false,"embedding":[-0.022859422,0.34394914,-0.26828787,-0.2738084,-0.63775945,-0.07745881,-1.0741392,0.1560609,0.37143725,0.019180652,0.20704071,0.27364534,-0.0067869863,0.52876467,0.6414323,-0.33678514,0.050067496,-0.93635535,-1.4170387,-0.11435936,-0.23238567,1.0892224,0.2120105,0.5584683,0.12247756,-0.064977966,0.14893794,-0.5124398,-0.96074337,-2.0346339,0.73753834,0.45415106,-0.08883569,-0.7438674,0.48413786,-0.58480775,0.05941936,-0.5025951,-0.18222712,-0.3229596,0.09404269,-0.6394208,-0.13328093,-0.35516846,0.26403305,0.2549522,0.011935851,-0.10359208,0.7161654,0.2805506,-0.3459439,-0.7086845,-0.3280931,-0.16880323,-0.38078505,-0.05401938,-0.40572488,0.0827634,0.102821976,0.112446666,-0.2216977,0.31631678,-4.125646,0.27062064,0.5810081,0.39316595,-0.09006638,-0.13725893,-0.23155731,0.06299147,0.22843449,-0.21436238,-0.16343841,-0.28368253,-0.07376786,-0.7834173,0.62592435,-0.18950966,-0.071794435,-0.8110028,0.026290458,0.8741538,0.0618044,0.5859047,-0.556113,0.3412014,0.046755597,-0.41176373,0.313688,-0.0122999,0.368527,-0.90580493,-0.079653844,0.5979872,-0.4543758,0.042234086,-0.10474573,0.24580964,0.1175344,3.1755562,0.80056226,-0.023514325,0.79566926,-0.7410976,-0.0014581755,-0.31112045,-0.039167665,0.100143805,0.5764502,0.1579162,0.65378535,-0.5400134,-0.020097598,0.018258024,0.35735926,-0.64368653,-0.84609747,-0.22342974,0.7505621,0.82978624,-0.4969715,0.5932594,-0.00894372,-0.20878446,0.1974807,0.16078371,-0.15507145,0.3443752,0.18211538,0.038514256,0.48453322,-0.36101848,-0.5905768,0.3267346,0.702126,0.23804805,0.14942536,-0.78500104,0.34118044,-0.7625391,0.2017467,-1.0103437,0.11651488,0.5579052,0.3436056,0.52296245,0.08520118,0.64077246,-0.4686281,-0.30036545,0.123490244,0.44900888,-0.48335668,-0.42536774,0.74946654,-0.068822786,-0.40755105,-0.24519834,-0.1692692,0.52134687,-0.45200285,-0.60263264,0.27810538,0.7042316,0.18795288,-0.3493522,0.046734825,0.22792357,0.6453644,0.451598,0.4749537,-0.23558003,-0.091785856,-0.3761208,0.19023523,-0.013168387,-0.27135915,-0.5617338,-0.0695993,-0.65293926,0.3943035,0.48974362,-0.5824184,-0.4601611,-0.002162248,0.28777352,0.3323654,0.29203975,0.8401477,0.75948477,-0.2863765,1.5633736,-1.0324217,-0.43823823,-0.39899313,-0.14089474,0.17630441,0.6572716,0.3455184,0.090465,0.023112308,-0.39716557,0.4068057,-0.13822418,-0.35577852,-0.021364532,0.4544103,0.026246114,0.18069449,-0.44668064,0.2769035,-0.73281,1.1664879,0.2932155,0.4086936,-0.35218203,-0.25461483,0.409438,0.24547005,0.5120449,0.1091405,0.28187823,-0.19503851,-0.47988772,-0.4020756,0.26754946,0.1451346,0.028484786,-0.5201925,-0.12774691,0.6918011,0.4549158,0.03442505,1.0406747,1.0184767,0.19375554,-0.2703382,0.8375021,0.20192006,-0.63289624,0.19444388,-0.16417834,-0.13579649,-0.2540757,-0.23970604,-1.1852467,-0.13074327,0.33187962,0.46423656,1.7475896,0.77317226,0.50678706,0.6598569,0.04874261,0.2757128,0.064221054,-2.202051,-0.26725778,-0.23615283,1.212763,-0.17240548,0.4422012,0.12729892,0.022317126,-0.2723784,-0.37636,-0.43468815,-1.0617322,-0.35068285,0.31613588,-0.35614374,0.45097005,0.30068332,-0.043214343,0.01916102,0.5010773,0.62649405,-0.31871548,-0.7229701,-0.6690801,0.32861578,-0.17093404,0.5118524,0.00037259609,-0.11062399,-0.35906035,0.13406049,0.44416454,-0.068312965,0.27592757,0.051042564,-0.49447364,-0.063102454,0.8905604,1.6143283,-0.1856372,-0.05961505,-0.35624927,-0.5815494,-0.21133251,-0.23919454,0.3232051,-0.13867204,-0.22855097,-0.93279076,-0.29881087,0.47200906,-0.49601215,-0.15198481,0.017862234,-0.32360247,-0.07765387,0.09425101,-0.24398094,0.6091076,-0.5912474,-0.2593696,0.047237284,-0.20522922,-1.7102749,-0.5065819,0.004834657,0.13834465,-0.19261189,-0.6813854,0.93593514,-0.1833483,-0.16403382,-0.21518946,1.5840325,0.62321234,-0.6040731,-0.65227884,0.421814,0.659497,-0.17027326,-0.077558756,-0.46503827,-0.67175186,0.034123674,0.9178276,1.7945781,0.045439854,0.014577113,0.19752969,0.25004318,-0.85095567,-1.5589556,-0.16364035,-0.26558465,-0.16827886,0.29318962,-0.6095997,-0.1322887,0.52664423,0.69676834,-0.31061935,-0.06737543,-0.7940618,2.265823,0.15195686,0.057539836,-0.032563098,-0.12275851,-0.060634516,0.18676114,0.72584385,-0.5401763,-0.56609005,-0.29295185,0.26685232,-0.19534579,0.35730702,0.4106815,0.62266856,0.33912587,0.23005708,0.6000891,0.2057446,0.54149806,0.43303984,0.21790418,-0.47692624,0.19675124],"result":{"type":"object","properties":{"markdown":{"description":"The Markdown content converted from the web page","type":"string"}},"required":["markdown"]},"sql_tables":null,"sql_queries":null,"file_inbox":null,"oauth":null,"assets":null,"runner":"any","operating_system":["linux","macos","windows"],"tool_set":""},false]} \ No newline at end of file diff --git a/tools/download-page/tool.ts b/tools/download-page/tool.ts index 80e36b63..13830abd 100644 --- a/tools/download-page/tool.ts +++ b/tools/download-page/tool.ts @@ -59,10 +59,14 @@ export const run: Run = async ( await browser.close(); console.log('Saving HTML to file...'); - Deno.writeTextFileSync(await getHomePath() + '/download-page.html', html); + const safeFileName = parameters.url.replace(/\./g, '_').replace(/[^a-zA-Z0-9_-]/g, ''); + const safeDate = new Date().toISOString().replace(/[:.]/g, '-'); + Deno.writeTextFileSync(await getHomePath() + `/raw.${safeFileName}.${safeDate}.html`, html); console.log('Converting HTML to Markdown...'); const turndownService = new TurndownService(); - const markdown = turndownService.turndown(html); + const markdown = turndownService + .remove(['script', 'style', 'title']) + .turndown(html); return Promise.resolve({ markdown }); };