Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove scripts #172

Merged
merged 1 commit into from
Feb 12, 2025
Merged

remove scripts #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions scripts/tests/download-page.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
const code = Deno.readTextFileSync(import.meta.dirname + '/../../tools/download-page/tool.ts');
const metadata = JSON.parse(Deno.readTextFileSync(import.meta.dirname + '/../../tools/download-page/metadata.json'));

type INPUTS = {
url: string;
};
const X_SHINKAI_TOOL_ID = `example-${Math.random().toString(36).substring(2, 15)}`;
const X_SHINKAI_APP_ID = `run-${Math.random().toString(36).substring(2, 15)}`;

const base_url = Deno.env.get('SHINKAI_NODE_ADDR') ?? 'http://localhost:9950';
const token = Deno.env.get('BEARER_TOKEN') ?? 'debug';
const llm_provider = Deno.env.get('INITIAL_AGENT_NAMES') ? (Deno.env.get('INITIAL_AGENT_NAMES')??'').split(',')[0] : 'llama3_1_8b';
async function runCommandTest(parameters: INPUTS) {
const response = await fetch(base_url + '/v2/code_execution', {
method: 'POST',
headers: {
'Authorization': 'Bearer ' + token,
'x-shinkai-tool-id': X_SHINKAI_TOOL_ID,
'x-shinkai-app-id': X_SHINKAI_APP_ID,
'x-shinkai-llm-provider': llm_provider,
'Content-Type': 'application/json; charset=utf-8',
},
body: JSON.stringify({
code,
tool_type: 'denodynamic',
llm_provider,
tools: metadata.tools || [],
parameters
})
});

const data = await response.json();
return data;
}

export async function downloadPageTest() {
const expect = (data: any, message: string | undefined, errorStr: string | undefined) => {
if (!data) throw Error('[Check failed] ' + (message ?? '') + ' ' + (errorStr ?? ''));
else console.log('[Check passed] ' + (message ?? ''));
}

const parametersA: INPUTS = {
url: 'https://shinkai.com'
}
const dataA= await runCommandTest(parametersA);
expect(dataA.markdown.match(/open.source/i), 'Page should mention open source', `Missing text on ${dataA.markdown.substring(0, 100)}...`);
}

// downloadPageTest().then(console.log).catch(console.error);
15 changes: 12 additions & 3 deletions scripts/zip.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ import { getMetadata, processToolsDirectory, saveToolsInNode } from "./build_too
import { join } from "https://deno.land/std/path/mod.ts";
import { exists } from "https://deno.land/std/fs/mod.ts";
import { stripVersion, systemTools } from "./build_tools/system.ts";
import { memoryTest } from "./tests/tool-memory.test.ts";
import { downloadPageTest } from "./tests/download-page.test.ts";

const TOOL_TESTS = !!Deno.env.get("TOOL_TESTS");

Deno.test("Check if related tools exist", async () => {
const tools_raw = await processToolsDirectory();
Expand Down Expand Up @@ -68,6 +72,11 @@ Deno.test("Compare shinkai-node generated ZIP __tool.json vs .tool-dump.test.jso
});


// Deno.test("Check if memory is working", async () => {
// await memoryTest();
// });
if (TOOL_TESTS) {
Deno.test("Check if memory is working", async () => {
await memoryTest();
});
Deno.test("Check if download-page is working", async () => {
await downloadPageTest();
});
}
2 changes: 1 addition & 1 deletion tools/download-page/.tool-dump.test.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"type":"Deno","content":[{"name":"Download Pages","homepage":null,"author":"@@official.shinkai","version":"1.0.0","js_code":"\nimport chromePaths from 'npm:chrome-paths@1.0.1';\nimport TurndownService from 'npm:turndown@7.2.0';\nimport { addExtra } from 'npm:puppeteer-extra@3.3.6';\nimport rebrowserPuppeteer from 'npm:rebrowser-puppeteer@23.10.1';\nimport StealthPlugin from 'npm:puppeteer-extra-plugin-stealth@2.11.2';\n\nimport { getHomePath } from './shinkai-local-support.ts';\n\ntype Configurations = {\n chromePath?: string;\n};\n\ntype Parameters = {\n url: string;\n incognito?: boolean;\n};\n\ntype Result = { markdown: string };\n\nexport type Run<C extends Record<string, any>, I extends Record<string, any>, R extends Record<string, any>> = (config: C, inputs: I) => Promise<R>;\n\nconst puppeteer = addExtra(rebrowserPuppeteer as any);\nconst pluginStealth = StealthPlugin();\npluginStealth.enabledEvasions.delete('chrome.loadTimes');\npluginStealth.enabledEvasions.delete('chrome.runtime');\npuppeteer.use(pluginStealth);\n\nexport const run: Run<Configurations, Parameters, Result> = async (\n configurations: Configurations,\n parameters: Parameters,\n): Promise<Result> => {\n const chromePath =\n configurations?.chromePath ||\n Deno.env.get('CHROME_PATH') ||\n chromePaths.chrome ||\n chromePaths.chromium;\n if (!chromePath) {\n throw new Error('Chrome path not found');\n }\n console.log({ chromePath })\n const browser = await puppeteer.launch({\n executablePath: chromePath,\n args: ['--disable-blink-features=AutomationControlled'],\n });\n\n const page = await browser.newPage();\n \n console.log(\"Navigating to website...\");\n await page.goto(parameters.url);\n \n console.log('Waiting for the page to load...');\n await page.waitForNetworkIdle();\n \n console.log('Extracting HTML content...');\n const html = await page.content();\n\n console.log('Closing browser...');\n await browser.close();\n\n console.log('Saving HTML to file...');\n Deno.writeTextFileSync(await getHomePath() + '/download-page.html', html);\n\n console.log('Converting HTML to Markdown...');\n const turndownService = new TurndownService();\n const markdown = turndownService.turndown(html);\n return Promise.resolve({ markdown });\n};\n","tools":[],"config":[{"BasicConfig":{"key_name":"chromePath","description":"The path to the Chrome executable","required":false,"type":null,"key_value":null}}],"description":"Downloads a URL and converts its HTML content to Markdown","keywords":["HTML to Markdown","web page downloader","content conversion","URL to Markdown"],"input_args":{"type":"object","properties":{"url":{"type":"string","description":"A URL of a web page to download"}},"required":["url"]},"output_arg":{"json":""},"activated":false,"embedding":[-0.022859422,0.34394914,-0.26828787,-0.2738084,-0.63775945,-0.07745881,-1.0741392,0.1560609,0.37143725,0.019180652,0.20704071,0.27364534,-0.0067869863,0.52876467,0.6414323,-0.33678514,0.050067496,-0.93635535,-1.4170387,-0.11435936,-0.23238567,1.0892224,0.2120105,0.5584683,0.12247756,-0.064977966,0.14893794,-0.5124398,-0.96074337,-2.0346339,0.73753834,0.45415106,-0.08883569,-0.7438674,0.48413786,-0.58480775,0.05941936,-0.5025951,-0.18222712,-0.3229596,0.09404269,-0.6394208,-0.13328093,-0.35516846,0.26403305,0.2549522,0.011935851,-0.10359208,0.7161654,0.2805506,-0.3459439,-0.7086845,-0.3280931,-0.16880323,-0.38078505,-0.05401938,-0.40572488,0.0827634,0.102821976,0.112446666,-0.2216977,0.31631678,-4.125646,0.27062064,0.5810081,0.39316595,-0.09006638,-0.13725893,-0.23155731,0.06299147,0.22843449,-0.21436238,-0.16343841,-0.28368253,-0.07376786,-0.7834173,0.62592435,-0.18950966,-0.071794435,-0.8110028,0.026290458,0.8741538,0.0618044,0.5859047,-0.556113,0.3412014,0.046755597,-0.41176373,0.313688,-0.0122999,0.368527,-0.90580493,-0.079653844,0.5979872,-0.4543758,0.042234086,-0.10474573,0.24580964,0.1175344,3.1755562,0.80056226,-0.023514325,0.79566926,-0.7410976,-0.0014581755,-0.31112045,-0.039167665,0.100143805,0.5764502,0.1579162,0.65378535,-0.5400134,-0.020097598,0.018258024,0.35735926,-0.64368653,-0.84609747,-0.22342974,0.7505621,0.82978624,-0.4969715,0.5932594,-0.00894372,-0.20878446,0.1974807,0.16078371,-0.15507145,0.3443752,0.18211538,0.038514256,0.48453322,-0.36101848,-0.5905768,0.3267346,0.702126,0.23804805,0.14942536,-0.78500104,0.34118044,-0.7625391,0.2017467,-1.0103437,0.11651488,0.5579052,0.3436056,0.52296245,0.08520118,0.64077246,-0.4686281,-0.30036545,0.123490244,0.44900888,-0.48335668,-0.42536774,0.74946654,-0.068822786,-0.40755105,-0.24519834,-0.1692692,0.52134687,-0.45200285,-0.60263264,0.27810538,0.7042316,0.18795288,-0.3493522,0.046734825,0.22792357,0.6453644,0.451598,0.4749537,-0.23558003,-0.091785856,-0.3761208,0.19023523,-0.013168387,-0.27135915,-0.5617338,-0.0695993,-0.65293926,0.3943035,0.48974362,-0.5824184,-0.4601611,-0.002162248,0.28777352,0.3323654,0.29203975,0.8401477,0.75948477,-0.2863765,1.5633736,-1.0324217,-0.43823823,-0.39899313,-0.14089474,0.17630441,0.6572716,0.3455184,0.090465,0.023112308,-0.39716557,0.4068057,-0.13822418,-0.35577852,-0.021364532,0.4544103,0.026246114,0.18069449,-0.44668064,0.2769035,-0.73281,1.1664879,0.2932155,0.4086936,-0.35218203,-0.25461483,0.409438,0.24547005,0.5120449,0.1091405,0.28187823,-0.19503851,-0.47988772,-0.4020756,0.26754946,0.1451346,0.028484786,-0.5201925,-0.12774691,0.6918011,0.4549158,0.03442505,1.0406747,1.0184767,0.19375554,-0.2703382,0.8375021,0.20192006,-0.63289624,0.19444388,-0.16417834,-0.13579649,-0.2540757,-0.23970604,-1.1852467,-0.13074327,0.33187962,0.46423656,1.7475896,0.77317226,0.50678706,0.6598569,0.04874261,0.2757128,0.064221054,-2.202051,-0.26725778,-0.23615283,1.212763,-0.17240548,0.4422012,0.12729892,0.022317126,-0.2723784,-0.37636,-0.43468815,-1.0617322,-0.35068285,0.31613588,-0.35614374,0.45097005,0.30068332,-0.043214343,0.01916102,0.5010773,0.62649405,-0.31871548,-0.7229701,-0.6690801,0.32861578,-0.17093404,0.5118524,0.00037259609,-0.11062399,-0.35906035,0.13406049,0.44416454,-0.068312965,0.27592757,0.051042564,-0.49447364,-0.063102454,0.8905604,1.6143283,-0.1856372,-0.05961505,-0.35624927,-0.5815494,-0.21133251,-0.23919454,0.3232051,-0.13867204,-0.22855097,-0.93279076,-0.29881087,0.47200906,-0.49601215,-0.15198481,0.017862234,-0.32360247,-0.07765387,0.09425101,-0.24398094,0.6091076,-0.5912474,-0.2593696,0.047237284,-0.20522922,-1.7102749,-0.5065819,0.004834657,0.13834465,-0.19261189,-0.6813854,0.93593514,-0.1833483,-0.16403382,-0.21518946,1.5840325,0.62321234,-0.6040731,-0.65227884,0.421814,0.659497,-0.17027326,-0.077558756,-0.46503827,-0.67175186,0.034123674,0.9178276,1.7945781,0.045439854,0.014577113,0.19752969,0.25004318,-0.85095567,-1.5589556,-0.16364035,-0.26558465,-0.16827886,0.29318962,-0.6095997,-0.1322887,0.52664423,0.69676834,-0.31061935,-0.06737543,-0.7940618,2.265823,0.15195686,0.057539836,-0.032563098,-0.12275851,-0.060634516,0.18676114,0.72584385,-0.5401763,-0.56609005,-0.29295185,0.26685232,-0.19534579,0.35730702,0.4106815,0.62266856,0.33912587,0.23005708,0.6000891,0.2057446,0.54149806,0.43303984,0.21790418,-0.47692624,0.19675124],"result":{"type":"object","properties":{"markdown":{"description":"The Markdown content converted from the web page","type":"string"}},"required":["markdown"]},"sql_tables":null,"sql_queries":null,"file_inbox":null,"oauth":null,"assets":null,"runner":"any","operating_system":["linux","macos","windows"],"tool_set":""},false]}
{"type":"Deno","content":[{"name":"Download Pages","homepage":null,"author":"@@official.shinkai","version":"1.0.0","js_code":"\nimport chromePaths from 'npm:chrome-paths@1.0.1';\nimport TurndownService from 'npm:turndown@7.2.0';\nimport { addExtra } from 'npm:puppeteer-extra@3.3.6';\nimport rebrowserPuppeteer from 'npm:rebrowser-puppeteer@23.10.1';\nimport StealthPlugin from 'npm:puppeteer-extra-plugin-stealth@2.11.2';\n\nimport { getHomePath } from './shinkai-local-support.ts';\n\ntype Configurations = {\n chromePath?: string;\n};\n\ntype Parameters = {\n url: string;\n incognito?: boolean;\n};\n\ntype Result = { markdown: string };\n\nexport type Run<C extends Record<string, any>, I extends Record<string, any>, R extends Record<string, any>> = (config: C, inputs: I) => Promise<R>;\n\nconst puppeteer = addExtra(rebrowserPuppeteer as any);\nconst pluginStealth = StealthPlugin();\npluginStealth.enabledEvasions.delete('chrome.loadTimes');\npluginStealth.enabledEvasions.delete('chrome.runtime');\npuppeteer.use(pluginStealth);\n\nexport const run: Run<Configurations, Parameters, Result> = async (\n configurations: Configurations,\n parameters: Parameters,\n): Promise<Result> => {\n const chromePath =\n configurations?.chromePath ||\n Deno.env.get('CHROME_PATH') ||\n chromePaths.chrome ||\n chromePaths.chromium;\n if (!chromePath) {\n throw new Error('Chrome path not found');\n }\n console.log({ chromePath })\n const browser = await puppeteer.launch({\n executablePath: chromePath,\n args: ['--disable-blink-features=AutomationControlled'],\n });\n\n const page = await browser.newPage();\n \n console.log(\"Navigating to website...\");\n await page.goto(parameters.url);\n \n console.log('Waiting for the page to load...');\n await page.waitForNetworkIdle();\n \n console.log('Extracting HTML content...');\n const html = await page.content();\n\n console.log('Closing browser...');\n await browser.close();\n\n console.log('Saving HTML to file...');\n const safeFileName = parameters.url.replace(/\\./g, '_').replace(/[^a-zA-Z0-9_-]/g, '');\n const safeDate = new Date().toISOString().replace(/[:.]/g, '-');\n Deno.writeTextFileSync(await getHomePath() + `/raw.${safeFileName}.${safeDate}.html`, html);\n\n console.log('Converting HTML to Markdown...');\n const turndownService = new TurndownService();\n const markdown = turndownService\n .remove(['script', 'style', 'title'])\n .turndown(html);\n return Promise.resolve({ markdown });\n};\n","tools":[],"config":[{"BasicConfig":{"key_name":"chromePath","description":"The path to the Chrome executable","required":false,"type":null,"key_value":null}}],"description":"Downloads a URL and converts its HTML content to Markdown","keywords":["HTML to Markdown","web page downloader","content conversion","URL to Markdown"],"input_args":{"type":"object","properties":{"url":{"type":"string","description":"A URL of a web page to download"}},"required":["url"]},"output_arg":{"json":""},"activated":false,"embedding":[-0.022859422,0.34394914,-0.26828787,-0.2738084,-0.63775945,-0.07745881,-1.0741392,0.1560609,0.37143725,0.019180652,0.20704071,0.27364534,-0.0067869863,0.52876467,0.6414323,-0.33678514,0.050067496,-0.93635535,-1.4170387,-0.11435936,-0.23238567,1.0892224,0.2120105,0.5584683,0.12247756,-0.064977966,0.14893794,-0.5124398,-0.96074337,-2.0346339,0.73753834,0.45415106,-0.08883569,-0.7438674,0.48413786,-0.58480775,0.05941936,-0.5025951,-0.18222712,-0.3229596,0.09404269,-0.6394208,-0.13328093,-0.35516846,0.26403305,0.2549522,0.011935851,-0.10359208,0.7161654,0.2805506,-0.3459439,-0.7086845,-0.3280931,-0.16880323,-0.38078505,-0.05401938,-0.40572488,0.0827634,0.102821976,0.112446666,-0.2216977,0.31631678,-4.125646,0.27062064,0.5810081,0.39316595,-0.09006638,-0.13725893,-0.23155731,0.06299147,0.22843449,-0.21436238,-0.16343841,-0.28368253,-0.07376786,-0.7834173,0.62592435,-0.18950966,-0.071794435,-0.8110028,0.026290458,0.8741538,0.0618044,0.5859047,-0.556113,0.3412014,0.046755597,-0.41176373,0.313688,-0.0122999,0.368527,-0.90580493,-0.079653844,0.5979872,-0.4543758,0.042234086,-0.10474573,0.24580964,0.1175344,3.1755562,0.80056226,-0.023514325,0.79566926,-0.7410976,-0.0014581755,-0.31112045,-0.039167665,0.100143805,0.5764502,0.1579162,0.65378535,-0.5400134,-0.020097598,0.018258024,0.35735926,-0.64368653,-0.84609747,-0.22342974,0.7505621,0.82978624,-0.4969715,0.5932594,-0.00894372,-0.20878446,0.1974807,0.16078371,-0.15507145,0.3443752,0.18211538,0.038514256,0.48453322,-0.36101848,-0.5905768,0.3267346,0.702126,0.23804805,0.14942536,-0.78500104,0.34118044,-0.7625391,0.2017467,-1.0103437,0.11651488,0.5579052,0.3436056,0.52296245,0.08520118,0.64077246,-0.4686281,-0.30036545,0.123490244,0.44900888,-0.48335668,-0.42536774,0.74946654,-0.068822786,-0.40755105,-0.24519834,-0.1692692,0.52134687,-0.45200285,-0.60263264,0.27810538,0.7042316,0.18795288,-0.3493522,0.046734825,0.22792357,0.6453644,0.451598,0.4749537,-0.23558003,-0.091785856,-0.3761208,0.19023523,-0.013168387,-0.27135915,-0.5617338,-0.0695993,-0.65293926,0.3943035,0.48974362,-0.5824184,-0.4601611,-0.002162248,0.28777352,0.3323654,0.29203975,0.8401477,0.75948477,-0.2863765,1.5633736,-1.0324217,-0.43823823,-0.39899313,-0.14089474,0.17630441,0.6572716,0.3455184,0.090465,0.023112308,-0.39716557,0.4068057,-0.13822418,-0.35577852,-0.021364532,0.4544103,0.026246114,0.18069449,-0.44668064,0.2769035,-0.73281,1.1664879,0.2932155,0.4086936,-0.35218203,-0.25461483,0.409438,0.24547005,0.5120449,0.1091405,0.28187823,-0.19503851,-0.47988772,-0.4020756,0.26754946,0.1451346,0.028484786,-0.5201925,-0.12774691,0.6918011,0.4549158,0.03442505,1.0406747,1.0184767,0.19375554,-0.2703382,0.8375021,0.20192006,-0.63289624,0.19444388,-0.16417834,-0.13579649,-0.2540757,-0.23970604,-1.1852467,-0.13074327,0.33187962,0.46423656,1.7475896,0.77317226,0.50678706,0.6598569,0.04874261,0.2757128,0.064221054,-2.202051,-0.26725778,-0.23615283,1.212763,-0.17240548,0.4422012,0.12729892,0.022317126,-0.2723784,-0.37636,-0.43468815,-1.0617322,-0.35068285,0.31613588,-0.35614374,0.45097005,0.30068332,-0.043214343,0.01916102,0.5010773,0.62649405,-0.31871548,-0.7229701,-0.6690801,0.32861578,-0.17093404,0.5118524,0.00037259609,-0.11062399,-0.35906035,0.13406049,0.44416454,-0.068312965,0.27592757,0.051042564,-0.49447364,-0.063102454,0.8905604,1.6143283,-0.1856372,-0.05961505,-0.35624927,-0.5815494,-0.21133251,-0.23919454,0.3232051,-0.13867204,-0.22855097,-0.93279076,-0.29881087,0.47200906,-0.49601215,-0.15198481,0.017862234,-0.32360247,-0.07765387,0.09425101,-0.24398094,0.6091076,-0.5912474,-0.2593696,0.047237284,-0.20522922,-1.7102749,-0.5065819,0.004834657,0.13834465,-0.19261189,-0.6813854,0.93593514,-0.1833483,-0.16403382,-0.21518946,1.5840325,0.62321234,-0.6040731,-0.65227884,0.421814,0.659497,-0.17027326,-0.077558756,-0.46503827,-0.67175186,0.034123674,0.9178276,1.7945781,0.045439854,0.014577113,0.19752969,0.25004318,-0.85095567,-1.5589556,-0.16364035,-0.26558465,-0.16827886,0.29318962,-0.6095997,-0.1322887,0.52664423,0.69676834,-0.31061935,-0.06737543,-0.7940618,2.265823,0.15195686,0.057539836,-0.032563098,-0.12275851,-0.060634516,0.18676114,0.72584385,-0.5401763,-0.56609005,-0.29295185,0.26685232,-0.19534579,0.35730702,0.4106815,0.62266856,0.33912587,0.23005708,0.6000891,0.2057446,0.54149806,0.43303984,0.21790418,-0.47692624,0.19675124],"result":{"type":"object","properties":{"markdown":{"description":"The Markdown content converted from the web page","type":"string"}},"required":["markdown"]},"sql_tables":null,"sql_queries":null,"file_inbox":null,"oauth":null,"assets":null,"runner":"any","operating_system":["linux","macos","windows"],"tool_set":""},false]}
8 changes: 6 additions & 2 deletions tools/download-page/tool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,14 @@ export const run: Run<Configurations, Parameters, Result> = async (
await browser.close();

console.log('Saving HTML to file...');
Deno.writeTextFileSync(await getHomePath() + '/download-page.html', html);
const safeFileName = parameters.url.replace(/\./g, '_').replace(/[^a-zA-Z0-9_-]/g, '');
const safeDate = new Date().toISOString().replace(/[:.]/g, '-');
Deno.writeTextFileSync(await getHomePath() + `/raw.${safeFileName}.${safeDate}.html`, html);

console.log('Converting HTML to Markdown...');
const turndownService = new TurndownService();
const markdown = turndownService.turndown(html);
const markdown = turndownService
.remove(['script', 'style', 'title'])
.turndown(html);
return Promise.resolve({ markdown });
};
Loading