Merge pull request #43 from dcSpark/feature/migrate-ts-upload-to-db

Enhance build process and tool management.
dcSpark · Jan 9, 2025 · f78b371 · f78b371
2 parents fbfb3fe + b3788ca
commit f78b371
Show file tree

Hide file tree

Showing 6 changed files with 614 additions and 11 deletions.
diff --git a/.github/workflows/build_tools.yaml b/.github/workflows/build_tools.yaml
@@ -24,15 +24,15 @@ jobs:
       - name: Install b3sum
         run: cargo install b3sum
 
+      - name: Setup Deno
+        uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y curl jq coreutils
-
-      - name: Make script executable
-        run: |
-          chmod +x ./.github/build_tool.sh
-          chmod +x ./.github/run_node.sh
+          sudo apt-get install -y curl jq zip unzip coreutils
 
       - name: Run node in background
         id: run_node
@@ -48,13 +48,15 @@ jobs:
           INITIALIZATION_DATA: ${{ secrets.INITIALIZATION_DATA }}
           DOWNLOAD_PREFIX: "https://download.shinkai.com/tools"
           SKIP_IMPORT_FROM_DIRECTORY: true
+          SHINKAI_STORE_ADDR: "https://shinkai-store-302883622007.us-central1.run.app"
+          SHINKAI_STORE_TOKEN: ${{ secrets.SHINKAI_STORE_TOKEN }}
         run: |
-          ./.github/run_node.sh &
+          ./scripts/run_node.sh &
           timeout 60 bash -c 'until curl -s --location "$SHINKAI_NODE_ADDR/v2/health_check" | jq -e ".status == \"ok\"" > /dev/null; do sleep 1; done'
-            curl --location "$SHINKAI_NODE_ADDR/v2/initial_registration" \
-          --header 'Content-Type: application/json; charset=utf-8' \
+          curl --location "$SHINKAI_NODE_ADDR/v2/initial_registration" \
+            --header 'Content-Type: application/json; charset=utf-8' \
             --data "$INITIALIZATION_DATA"
-          ./.github/build_tool.sh
+          deno run --allow-read --allow-write --allow-env --allow-net --allow-run scripts/build_tools.ts
           if jq -e '.[] | select(.description == "")' packages/directory.json > /dev/null; then
             echo "Error: Empty descriptions found in packages/directory.json"
             exit 1

diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,4 @@ tools/**/shinkai-local-tools.ts
 .DS_Store
 packages**
 shinkai-node**
-registry/node_modules
+deno.lock
diff --git a/.github/build_tool.sh → scripts/build_tool.sh b/.github/build_tool.sh → scripts/build_tool.sh
diff --git a/scripts/build_tools.ts b/scripts/build_tools.ts
@@ -0,0 +1,315 @@
+import { join } from "https://deno.land/std/path/mod.ts";
+import { exists } from "https://deno.land/std/fs/mod.ts";
+import { encode } from "https://deno.land/std/encoding/base64.ts";
+
+interface Metadata {
+  id: string;
+  name: string;
+  description: string;
+  author: string;
+  version: string;
+  keywords: string[];
+  configurations?: {
+    properties: Record<string, any>;
+    required: string[];
+  };
+  parameters: {
+    properties: Record<string, any>;
+  };
+  result: Record<string, any>;
+}
+
+interface DirectoryEntry {
+  default?: boolean;
+  name: string;
+  author: string;
+  keywords: string[];
+  type: "Tool" | "Agent" | "Scheduled Task";
+  tool_language?: string;
+  version: string;
+  description: string;
+  router_key?: string;
+  hash: string;
+  file: string;
+  agent_id?: string;
+}
+
+async function getToolType(file: string): Promise<string> {
+  return file.endsWith(".ts") ? "Deno" : "Python";
+}
+
+async function calculateBlake3Hash(filePath: string): Promise<string> {
+  const command = new Deno.Command("b3sum", {
+    args: [filePath],
+  });
+  const { stdout } = await command.output();
+  const output = new TextDecoder().decode(stdout);
+  return output.split(" ")[0];
+}
+
+async function buildToolJson(toolContent: string, metadata: Metadata, toolType: string) {
+  const content = [{
+    activated: false,
+    assets: null,
+    file_inbox: null,
+    oauth: null,
+    output_arg: { json: "" },
+    author: metadata.author || "Unknown",
+    config: metadata.configurations?.properties ? 
+      Object.entries(metadata.configurations.properties).map(([key, value]) => ({
+        BasicConfig: {
+          key_name: key,
+          description: value.description || "",
+          required: metadata.configurations?.required?.includes(key) || false,
+          key_value: null
+        }
+      })) : [],
+    description: metadata.description || "No description provided.",
+    input_args: metadata.parameters || [],
+    keywords: metadata.keywords || [],
+    name: metadata.name || "Unknown",
+    result: metadata.result || {},
+    sql_queries: [],
+    sql_tables: [],
+    toolkit_name: metadata.id || "Unknown",
+    tools: [],
+    version: metadata.version || "1.0.0",
+    [toolType === "Python" ? "py_code" : "js_code"]: toolContent
+  }, false];
+
+  return { content, type: toolType };
+}
+
+async function processToolsDirectory() {
+  const tools: DirectoryEntry[] = [];
+
+  // Process tools
+  for await (const entry of Deno.readDir("tools")) {
+    if (!entry.isDirectory) continue;
+
+    const toolDir = join("tools", entry.name);
+    const toolName = entry.name;
+
+    console.log(`Processing ${toolName}...`);
+
+    // Find tool file
+    let toolFile = "";
+    if (await exists(join(toolDir, "tool.ts"))) {
+      toolFile = join(toolDir, "tool.ts");
+    } else if (await exists(join(toolDir, "tool.py"))) {
+      toolFile = join(toolDir, "tool.py");
+    }
+
+    if (!toolFile || !await exists(join(toolDir, "metadata.json"))) {
+      console.error(`Error: Missing required files in ${toolDir}`);
+      continue;
+    }
+
+    // Read files
+    const metadata: Metadata = JSON.parse(await Deno.readTextFile(join(toolDir, "metadata.json")));
+    const toolContent = await Deno.readTextFile(toolFile);
+    const toolType = await getToolType(toolFile);
+
+    // Build tool JSON
+    const toolJson = await buildToolJson(toolContent, metadata, toolType);
+
+    // Send to Shinkai node
+    const response = await fetch(`${Deno.env.get("SHINKAI_NODE_ADDR")}/v2/add_shinkai_tool`, {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${Deno.env.get("BEARER_TOKEN")}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(toolJson),
+    });
+
+    if (!response.ok) {
+      console.error(`Failed to upload tool to Shinkai node. HTTP status: ${response.status}`);
+      console.error(`Response: ${await response.text()}`);
+      continue;
+    }
+
+    const uploadedTool = await response.json();
+    const toolRouterKey = uploadedTool.message.replace(/.*key: /, "");
+
+    // Get tool zip
+    const zipResponse = await fetch(
+      `${Deno.env.get("SHINKAI_NODE_ADDR")}/v2/export_tool?tool_key_path=${toolRouterKey}`,
+      {
+        headers: {
+          "Authorization": `Bearer ${Deno.env.get("BEARER_TOKEN")}`,
+        },
+      }
+    );
+
+    if (!zipResponse.ok) {
+      console.error(`Failed to download zip for ${toolName}`);
+      continue;
+    }
+
+    // Save zip file
+    const zipPath = join("packages", `${toolName}.zip`);
+    await Deno.writeFile(zipPath, new Uint8Array(await zipResponse.arrayBuffer()));
+
+    // Validate zip
+    try {
+      const validateZip = new Deno.Command("unzip", {
+        args: ["-t", zipPath],
+      });
+      await validateZip.output();
+    } catch {
+      console.error(`Error: Invalid zip file downloaded for ${toolName}`);
+      await Deno.remove(zipPath);
+      continue;
+    }
+
+    // Calculate hash
+    const blake3Hash = await calculateBlake3Hash(zipPath);
+
+    // Check for .default file
+    const hasDefault = await exists(join(toolDir, ".default"));
+
+    tools.push({
+      default: hasDefault,
+      name: toolName,
+      author: metadata.author || "Unknown",
+      keywords: metadata.keywords || ["tool"],
+      type: "Tool", 
+      tool_language: toolType,
+      version: metadata.version || "0.0.0",
+      description: metadata.description || "No description provided.",
+      router_key: toolRouterKey,
+      hash: blake3Hash,
+      file: `${Deno.env.get("DOWNLOAD_PREFIX")}/${toolName}.zip`,
+    });
+  }
+
+  return tools;
+}
+
+async function processAgentsDirectory() {
+  const agents: DirectoryEntry[] = [];
+
+  // Process agents
+  for await (const entry of Deno.readDir("agents")) {
+    if (!entry.isFile || !entry.name.endsWith(".json")) continue;
+
+    console.log(`Processing agent ${entry.name}...`);
+
+    const agentContent = JSON.parse(await Deno.readTextFile(join("agents", entry.name)));
+    const agentId = agentContent.agent_id;
+
+    // Create zip
+    const zipPath = join("packages", `${agentId}.zip`);
+    const zip = new Deno.Command("zip", {
+      args: ["-j", zipPath, join("agents", entry.name)],
+    });
+    await zip.output();
+
+    const blake3Hash = await calculateBlake3Hash(zipPath);
+
+    agents.push({
+      name: agentContent.name,
+      author: agentContent.author || "Unknown",
+      keywords: agentContent.keywords || ["agent"],
+      type: "Agent",
+      version: agentContent.version || "0.0.0",
+      description: agentContent.ui_description,
+      hash: blake3Hash,
+      file: `${Deno.env.get("DOWNLOAD_PREFIX")}/${agentId}.zip`,
+      agent_id: agentId,
+    });
+  }
+
+  return agents;
+}
+
+async function processCronsDirectory() {
+  const crons: DirectoryEntry[] = [];
+
+  // Process crons
+  for await (const entry of Deno.readDir("crons")) {
+    if (!entry.isFile || !entry.name.endsWith(".json")) continue;
+
+    console.log(`Processing cron ${entry.name}...`);
+
+    const cronContent = JSON.parse(await Deno.readTextFile(join("crons", entry.name)));
+    const cronId = entry.name.replace(".json", "");
+
+    // Create zip
+    const zipPath = join("packages", `${cronId}.zip`);
+    const zip = new Deno.Command("zip", {
+      args: ["-j", zipPath, join("crons", entry.name)],
+    });
+    await zip.output();
+
+    const blake3Hash = await calculateBlake3Hash(zipPath);
+
+    crons.push({
+      name: cronContent.name,
+      author: cronContent.author || "Unknown",
+      keywords: cronContent.keywords || ["cron"],
+      type: "Scheduled Task",
+      version: cronContent.version || "0.0.0",
+      description: cronContent.description,
+      hash: blake3Hash,
+      file: `${Deno.env.get("DOWNLOAD_PREFIX")}/${cronId}.zip`,
+    });
+  }
+
+  return crons;
+}
+
+async function processProducts() {
+  // Create packages directory
+  await Deno.mkdir("packages", { recursive: true });
+
+  // Initialize empty directory.json
+  await Deno.writeTextFile("packages/directory.json", "[]");
+
+  // Process all directories in parallel
+  const [tools, agents, crons] = await Promise.all([
+    processToolsDirectory(),
+    processAgentsDirectory(),
+    processCronsDirectory()
+  ]);
+
+  // Write final directory.json
+  const directory = [...tools, ...agents, ...crons];
+  await Deno.writeTextFile("packages/directory.json", JSON.stringify(directory, null, 2));
+
+  // Upload directory.json to Shinkai Store
+  for (const entry of directory) {
+    let response = await fetch(`${Deno.env.get("SHINKAI_STORE_ADDR")}/store/products`, {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${Deno.env.get("SHINKAI_STORE_TOKEN")}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(entry),
+    });
+
+    if (response.status === 409) {
+      const responseBody = await response.text();
+      if (responseBody.includes("already exists")) {
+        // Product exists, use PUT endpoint instead
+        const putResponse = await fetch(`${Deno.env.get("SHINKAI_STORE_ADDR")}/store/products/${entry.router_key}`, {
+          method: "PUT", 
+          headers: {
+            "Authorization": `Bearer ${Deno.env.get("SHINKAI_STORE_TOKEN")}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify(entry),
+        });
+        response = putResponse;
+      }
+    }
+
+    console.log(`Upload to Store Response (${response.status}): ${await response.text()}`);
+  }
+}
+
+// Run the script
+if (import.meta.main) {
+  await processProducts();
+}