integration.ts

/**
 * These integration tests depend on a running MinIO installation.
 *
 * See the README for instructions.
 */
import { assert } from "@std/assert/assert";
import { assertEquals } from "@std/assert/equals";
import { assertInstanceOf } from "@std/assert/instance-of";
import { assertRejects } from "@std/assert/rejects";
import { S3Client, S3Errors } from "./mod.ts";

const config = {
  endPoint: "localhost",
  port: 9000,
  useSSL: false,
  region: "dev-region",
  accessKey: "AKIA_DEV",
  secretKey: "secretkey",
  bucket: "dev-bucket",
  pathStyle: true,
};
const client = new S3Client(config);

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Test an unauthenticated client downloading public data

Deno.test({
  name: "the API client can be used without authentication (this also tests SSL and pathStyle: false)",
  fn: async () => {
    const publicClient = new S3Client({
      endPoint: "s3.amazonaws.com",
      port: 443,
      useSSL: true,
      region: "us-east-1",
      bucket: "amazon-pqa",
      pathStyle: false,
    });
    const response = await publicClient.getObject("readme.txt").then((r) => r.text());
    const expected = await fetch("https://amazon-pqa.s3.amazonaws.com/readme.txt").then((r) => r.text());
    assertEquals(response, expected);
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Error parsing

Deno.test({
  name: "error parsing",
  fn: async () => {
    const unauthorizedClient = new S3Client({ ...config, secretKey: "invalid key" });
    const err = await assertRejects(
      () => unauthorizedClient.putObject("test.txt", "This is the contents of the file."),
    );
    assertInstanceOf(err, S3Errors.ServerError);
    assertEquals(err.statusCode, 403);
    assertEquals(err.code, "SignatureDoesNotMatch");
    assertEquals(
      err.message,
      "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
    );
    assertEquals(err.bucketName, config.bucket);
    // This used to work but MinIO no longer includes the region name in the error XML
    // assertEquals(err.region, config.region);
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// putObject()

Deno.test({
  name: "putObject() can upload a small file",
  fn: async () => {
    const response = await client.putObject("test.txt", "This is the contents of the file.");
    assertEquals(response.etag, "f6b64dbfb5d44e98363ff586e08f7fe6"); // The etag is generated by the server, based on the contents, so this confirms it worked.
  },
});

Deno.test({
  name: "putObject() can set metadata",
  fn: async () => {
    const key = "test-with-metadata.txt";
    const metadata = {
      "Content-Type": "text/plain",
      "Cache-Control": "public, max-age=456789, immutable",
      "x-amz-meta-custom-header": "This is a custom value",
    };
    await client.putObject(key, "This is the contents of the file.", { metadata });
    const stat = await client.statObject(key);
    assertEquals(stat.key, key);
    assertEquals(stat.metadata, metadata);
  },
});

Deno.test({
  name: "putObject() can stream a large file upload",
  fn: async () => {
    // First generate a 32MiB file in memory, 1 MiB at a time, as a stream
    let dataStream;
    if (typeof ReadableStream.from !== "undefined") {
      dataStream = ReadableStream.from(async function* () {
        for (let i = 0; i < 32; i++) {
          yield new Uint8Array(1024 * 1024).fill(i); // Yield 1MB of data
        }
      }());
    } else {
      // ReadableStream.from is not yet supported by some runtimes :/
      // https://github.com/oven-sh/bun/issues/3700
      // https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream/from_static#browser_compatibility
      let i = 0;
      dataStream = new ReadableStream({
        pull(controller) {
          if (i < 32) {
            controller.enqueue(new Uint8Array(1024 * 1024).fill(i++));
          } else controller.close();
        },
      });
    }

    // Upload the 32MB stream data as 7 5MB parts. The client doesn't know in advance how big the stream is.
    const key = "test-32m.dat";
    const metadata = { "Content-Type": "test/streaming", "x-amz-meta-custom-header": "This is a custom value!" };
    const response = await client.putObject(key, dataStream, { partSize: 5 * 1024 * 1024, metadata });
    // The etag is generated by the server, based on the contents. Also, etags for multi-part uploads are
    // different than for regular uploads, so the "-7" confirms it worked and used a multi-part upload.
    assertEquals(response.etag, "ca6d977b6e7dc87ab5c5892e124c7277-7");
    // Validate that the metadata was set:
    const stat = await client.statObject(key);
    assertEquals(stat.metadata, metadata);
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// exists()

Deno.test({
  name: "exists() can check if an object exists",
  fn: async () => {
    const result1 = await client.exists("definitely-does-not-exist.foobar");
    assertEquals(result1, false);
    await client.putObject("this-will-exist.now", "contents");
    const result2 = await client.exists("this-will-exist.now");
    assertEquals(result2, true);
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// deleteObject()

Deno.test({
  name: "deleteObject() can delete an object",
  fn: async () => {
    const key = "object-for-deletion-tests.txt";
    await client.putObject(key, "contents");
    assertEquals(await client.exists(key), true);
    await client.deleteObject(key);
    assertEquals(await client.exists(key), false);
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// statObject()

Deno.test({
  name: "statObject() can get an object's status",
  fn: async () => {
    const key = "test-stat.txt";
    const metadata = {
      "Content-Type": "test/fake-data",
      "Cache-Control": "public, max-age=456789, immutable",
      "x-amz-meta-custom-header": "This is a custom value!",
    };
    const contents = "This is the contents of the file. 🎈"; // Red balloon tests unicode support
    await client.putObject(key, contents, { metadata });
    const stat = await client.statObject(key);
    assertEquals(stat.type, "Object");
    assertEquals(stat.key, key);
    assertInstanceOf(stat.lastModified, Date);
    assertEquals(stat.lastModified.getFullYear(), new Date().getFullYear()); // This may fail at exactly midnight on New Year's, no big deal
    assertEquals(stat.size, new TextEncoder().encode(contents).length); // Size in bytes is different from the length of the string
    assertEquals(stat.versionId, null);
    assertEquals(stat.metadata, metadata);
  },
});

Deno.test({
  name: "statObject() can include custom headers in the request",
  fn: async () => {
    const key = "test-stat-with-custom-headers.txt";
    const contents = "Testing custom headers in statObject";
    // This is the base64 encoded SHA-256 checksum of the above contents.
    // toBase64(new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(contents))));
    const checksumSha256 = "le9+OIGqpyujFCiCz22BYwWIeGHRx6LX2UIhz9GyeSI=";
    const baseMetadata = {
      "Content-Type": "text/plain",
      "x-amz-meta-custom-header": "custom-value",
    };
    const metadataWithChecksum = {
      ...baseMetadata,
      "x-amz-checksum-sha256": checksumSha256,
    };
    await client.putObject(key, contents, { metadata: metadataWithChecksum });

    // Test without the checksum mode header
    const statWithout = await client.statObject(key);
    assertEquals(statWithout.type, "Object");
    assertEquals(statWithout.key, key);
    assertEquals(statWithout.metadata, baseMetadata);

    // Test WITH the checksum header - now the response should include the x-amz-checksum-sha256 header.
    const statWith = await client.statObject(key, {
      headers: {
        "x-amz-checksum-mode": "ENABLED",
      },
    });
    assertEquals(statWith.type, "Object");
    assertEquals(statWith.key, key);
    assertEquals(statWith.metadata, metadataWithChecksum);
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// getObject()

Deno.test({
  name: "getObject() can download a small file",
  fn: async () => {
    const contents = "This is the contents of the file. 👻"; // Throw in an Emoji to ensure Unicode round-trip is working.
    await client.putObject("test-get.txt", contents);
    const response = await client.getObject("test-get.txt");
    assertEquals(await response.text(), contents);
  },
});

Deno.test({
  name: "getPartialObject() can download a partial file",
  fn: async () => {
    await client.putObject("test-get2.txt", "This is the contents of the file. 👻");
    const response = await client.getPartialObject("test-get2.txt", { offset: 12, length: 8 });
    assertEquals(await response.text(), "contents");
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// non-ascii characters in URLs

for (
  const path of [
    "simple.txt",
    "файл/gemütlich.txt",
    "path with spaces.txt",
    "yes&no.dat",
    "foo(bar)",
    "1+1=2",
    "~backup<crazy>.foo",
  ]
) {
  Deno.test({
    name: `get/put/list with unicode or special characters in URLs: ${path}`,
    // only: true,
    fn: async () => {
      const prefix = `filenames-test-${(Math.random() + 1).toString(36).substring(7)}/`;
      const contents = `This is the contents of the file called '${path}'.`;
      await client.putObject(prefix + path, contents);
      const response = await client.getObject(prefix + path);
      assertEquals(await response.text(), contents);
      const keys = await Array.fromAsync(client.listObjects({ prefix }), (entry) => entry.key);
      assertEquals(keys, [prefix + path]);
    },
  });
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// presignedGetObject()

Deno.test({
  name: "presignedGetObject() can create a pre-signed URL to download a file.",
  fn: async () => {
    const contents = "This is the contents of the file. 👻"; // Throw in an Emoji to ensure Unicode round-trip is working.
    await client.putObject("test-presigned.cstm", contents);
    const presignedUrl = await client.presignedGetObject("test-presigned.cstm", {
      // Also try overriding a response parameter
      responseParams: { "response-content-type": "custom/content-type" },
    });
    // Now use the pre-signed URL to download the file
    const response = await fetch(presignedUrl);
    assertEquals(await response.text(), contents);
    assertEquals(await response.headers.get("Content-Type"), "custom/content-type");
  },
});

Deno.test({
  name: "presignedGetObject() with session token can be used to retrieve object content",
  fn: async () => {
    // Create a client with session token
    const clientWithToken = new S3Client({
      ...config,
      sessionToken: "test-session-token",
    });

    // Create a test file with unique content
    const key = "test-presigned-url-content.txt";
    const content = "This is unique content to verify retrieval: " + Date.now();
    await client.putObject(key, content);

    try {
      // Generate a presigned URL with the session token
      const presignedUrl = await clientWithToken.presignedGetObject(key);

      // Verify the URL format is correct
      assert(
        presignedUrl.includes("X-Amz-Security-Token=test-session-token"),
        "Presigned URL should include the session token",
      );

      // NOTE: We can't actually fetch the content using the presigned URL in this test
      // because we don't have a valid session token.
      // TODO: use the STS API to create and test with valid temporary credentials:
      // https://github.com/minio/minio/blob/master/docs/sts/assume-role.md#sample-post-request
    } finally {
      // Clean up
      await client.deleteObject(key);
    }
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// listObjects()

Deno.test({
  name: "listObjects() can return an empty list when no keys match the prefix",
  fn: async () => {
    const response = client.listObjects({ prefix: "NO MATCH" });
    assertEquals(await response.next(), { done: true, value: undefined });
  },
});

Deno.test({
  name: "listObjects() can return a flat list of objects under a certain prefix",
  fn: async () => {
    const prefix = "list-objects-test-1/";
    await client.putObject(`not-under-that-prefix.txt`, "file Zero");
    await client.putObject(`${prefix}file-a.txt`, "file A");
    await client.putObject(`${prefix}file-b.txt`, "file B");
    await client.putObject(`${prefix}subpath/file-c.txt`, "file C");
    await client.putObject(`${prefix}subpath/file-d.txt`, "file D");
    const response = client.listObjects({ prefix });
    const results = await Array.fromAsync(response);
    assertEquals(results.length, 4);
    assertEquals(results[0].key, "list-objects-test-1/file-a.txt");
    assertEquals(results[0].etag, "31d97c4d04593b21b399ace73b061c34");
    assertEquals(results[0].size, 6);
    assertEquals(results[0].type, "Object");
    assertEquals(results[0].lastModified instanceof Date, true);
    // This test may occasionally be flaky if run at the very instant we're changing to a new month
    // or a new year, but that's OK:
    assertEquals(results[0].lastModified.getFullYear(), new Date().getFullYear());
    assertEquals(results[0].lastModified.getMonth(), new Date().getMonth());

    assertEquals(results[1].key, "list-objects-test-1/file-b.txt");
    assertEquals(results[1].etag, "1651d570b74339e94cace90cde7d3147");
    assertEquals(results[2].key, "list-objects-test-1/subpath/file-c.txt");
    assertEquals(results[3].key, "list-objects-test-1/subpath/file-d.txt");
  },
});

Deno.test({
  name: "listObjects() can return a flat list of objects, spanning multiple pages",
  fn: async () => {
    const prefix = "list-objects-test-2/";
    // Create 30 files, in parallel
    const putPromises = [];
    for (let i = 0; i < 30; i++) {
      putPromises.push(client.putObject(`${prefix}file-${i < 10 ? "0" : ""}${i}.txt`, `file ${i} contents`));
    }
    await Promise.all(putPromises);
    // Now retrieve them:
    const response = client.listObjects({ prefix, pageSize: 10 });
    const results = await Array.fromAsync(response);
    assertEquals(results.length, 30);
    assertEquals(results[0].key, `${prefix}file-00.txt`);
    assertEquals(results[29].key, `${prefix}file-29.txt`);

    // And it can limit the total number of results:
    const limitedResponse = client.listObjects({ prefix, pageSize: 10, maxResults: 25 });
    const limitedResults = await Array.fromAsync(limitedResponse);
    assertEquals(limitedResults.length, 25);
  },
});

Deno.test({
  name: "listObjectsGrouped() can group results using a delimiter",
  fn: async () => {
    const prefix = "list-objects-test-3/";
    await client.putObject(`${prefix}file-a.txt`, "file A");
    await client.putObject(`${prefix}file-b.txt`, "file B");
    await client.putObject(`${prefix}subpath-1/file-1-a.txt`, "file 1A");
    await client.putObject(`${prefix}subpath-1/file-1-b.txt`, "file 1B");
    await client.putObject(`${prefix}subpath-2/file-2-a.txt`, "file 1A");
    await client.putObject(`${prefix}subpath-2/file-2-b.txt`, "file 1B");
    await client.putObject(`${prefix}x-file.txt`, "file X");

    const response = client.listObjectsGrouped({ prefix, delimiter: "/", pageSize: 3 });
    const results = await Array.fromAsync(response);
    assertEquals(results.length, 5);
    // Note the order that we get the results in:
    assert(results[0].type === "Object");
    assertEquals(results[0].key, `${prefix}file-a.txt`);
    assert(results[1].type === "Object");
    assertEquals(results[1].key, `${prefix}file-b.txt`);
    assert(results[2].type === "CommonPrefix");
    assertEquals(results[2].prefix, `${prefix}subpath-1/`);
    assert(results[3].type === "CommonPrefix");
    assertEquals(results[3].prefix, `${prefix}subpath-2/`);
    assert(results[4].type === "Object");
    assertEquals(results[4].key, `${prefix}x-file.txt`);
  },
});

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// copyObject()

Deno.test({
  name: "copyObject() can copy a file",
  fn: async () => {
    const contents = "This is the contents of the copy test file. 👻"; // Throw in an Emoji to ensure Unicode round-trip is working.
    const sourceKey = "test-copy-source.txt";
    const destKey = "test-copy-dest.txt";

    // Create the source file:
    const uploadResult = await client.putObject(sourceKey, contents);
    // Make sure the destination doesn't yet exist:
    await client.deleteObject(destKey);
    assertEquals(await client.exists(destKey), false);

    const response = await client.copyObject({ sourceKey }, destKey);
    assertEquals(uploadResult.etag, response.etag);
    assertEquals(uploadResult.versionId, response.copySourceVersionId);
    assertInstanceOf(response.lastModified, Date);

    // Download the file to confirm that the copy worked.
    const downloadResult = await client.getObject(destKey);
    assertEquals(await downloadResult.text(), contents);
  },
});

Deno.test({
  name: "copyObject() gives an appropriate error if the source file doesn't exist.",
  fn: async () => {
    const sourceKey = "non-existent-source";
    const err = await assertRejects(
      () => client.copyObject({ sourceKey }, "any-dest.txt"),
    );
    assertInstanceOf(err, S3Errors.ServerError);
    assertEquals(err.code, "NoSuchKey");
    assertEquals(err.statusCode, 404);
    assertEquals(err.key, sourceKey);
    assertEquals(err.message, "The specified key does not exist.");
  },
});

Deno.test({
  name: "copyObject() copies metadata, but we can override it if we want to",
  fn: async () => {
    const contents = new Uint8Array([1, 2, 3, 4, 5, 6]);
    const sourceKey = "test-copy-metadata-source.txt";
    const destKeySame = "test-copy-metadata-same.txt";
    const destKeyNew = "test-copy-metadata-new.txt";

    // Create the source file:
    const metadata = {
      "Content-Type": "test/custom",
      "x-amz-meta-custom-key": "custom-value",
    };
    await client.putObject(sourceKey, contents, { metadata });
    // Make sure the destination doesn't yet exist:
    await client.deleteObject(destKeySame);
    assertEquals(await client.exists(destKeySame), false);

    // Copy it with the same metadata:
    await client.copyObject({ sourceKey }, destKeySame);
    const stat = await client.statObject(destKeySame);
    assertEquals(stat.metadata, metadata);

    // Copy it with the different metadata:
    const newMetadata = {
      "Content-Type": "application/javascript",
      "x-amz-meta-other": "new",
    };
    await client.copyObject({ sourceKey }, destKeyNew, { metadata: newMetadata });
    const statNew = await client.statObject(destKeyNew);
    assertEquals(statNew.metadata, newMetadata);
  },
});

Deno.test({
  name: "bucketExists() can check if a bucket exists",
  fn: async () => {
    const testBucketName = "test-bucket";
    // Check if the bucket exists. It should not.
    let exists = await client.bucketExists(testBucketName);
    assertEquals(exists, false);

    // Create a bucket for testing and check if it exists
    await client.makeBucket(testBucketName);
    exists = await client.bucketExists(testBucketName);
    assertEquals(exists, true);

    // Delete the bucket and check if it exists
    await client.removeBucket(testBucketName);
    exists = await client.bucketExists(testBucketName);
    assertEquals(exists, false);
  },
});

Deno.test({
  name: "getPresignedUrl() correctly includes session token when it exists",
  fn: async () => {
    // Create a client with session token
    const clientWithToken = new S3Client({
      ...config,
      sessionToken: "test-session-token",
    });

    // Create a test file
    const key = "test-presigned-url-with-token.txt";
    const content = "This is test content for presigned URL with token";
    await client.putObject(key, content);

    // Generate a presigned URL with the session token
    const presignedUrl = await clientWithToken.presignedGetObject(key);

    // Verify the URL contains the session token
    assert(
      presignedUrl.includes("X-Amz-Security-Token=test-session-token"),
      "Presigned URL should include the session token",
    );

    // Clean up
    await client.deleteObject(key);
  },
});

Deno.test({
  name: "presignedPostObject",
  fn: async () => {
    const objectName = `test-presigned-post-${crypto.randomUUID()}`;
    const content = "hello world";
    const contentType = "text/plain";
    const metadata = {
      "Content-Type": contentType,
      "x-amz-meta-test": "test-value",
    };

    // 1. Basic presigned POST with metadata
    const { url, fields } = await client.presignedPostObject(objectName, {
      expirySeconds: 600,
      fields: metadata,
    });

    // Create form data for the upload
    const formData = new FormData();
    // Add all required fields from the presigned POST
    Object.entries(fields).forEach(([key, value]) => {
      formData.append(key, value);
    });
    // Add the file content
    formData.append("file", new Blob([content], { type: contentType }));

    // Upload the object using the presigned POST
    const response = await fetch(url, {
      method: "POST",
      body: formData,
    });

    assertEquals(response.status, 204);

    // Verify the object was uploaded correctly
    const getResponse = await client.getObject(objectName);
    const uploadedContent = await getResponse.text();
    assertEquals(uploadedContent, content);

    // Verify metadata was applied correctly
    const objectStat = await client.statObject(objectName);
    assertEquals(objectStat.metadata["Content-Type"], contentType);
    assertEquals(objectStat.metadata["x-amz-meta-test"], "test-value");

    // 2. Test with conditions - content length range
    const objectName2 = `test-presigned-post-conditions-${crypto.randomUUID()}`;
    const { url: url2, fields: fields2 } = await client.presignedPostObject(objectName2, {
      expirySeconds: 600,
      fields: { "Content-Type": contentType },
      conditions: [
        ["content-length-range", "1", "100"], // Content must be between 1 and 100 bytes
      ],
    });

    const formData2 = new FormData();
    Object.entries(fields2).forEach(([key, value]) => {
      formData2.append(key, value);
    });
    formData2.append("file", new Blob([content], { type: contentType }));

    const response2 = await fetch(url2, {
      method: "POST",
      body: formData2,
    });

    assertEquals(response2.status, 204);

    // Verify
    const getResponse2 = await client.getObject(objectName2);
    const uploadedContent2 = await getResponse2.text();
    assertEquals(uploadedContent2, content);

    // Clean up
    await client.deleteObject(objectName);
    await client.deleteObject(objectName2);
  },
});

Deno.test({
  name: "presignedPostObject - with policy condition violations",
  fn: async () => {
    const objectName = `test-presigned-post-error-${crypto.randomUUID()}`;
    const smallContent = "small";
    const largeContent = "x".repeat(200); // 200 characters, exceeds the 100 byte limit we'll set
    const contentType = "text/plain";

    // Create presigned POST with a strict content length restriction
    const { url, fields } = await client.presignedPostObject(objectName, {
      expirySeconds: 600,
      fields: { "Content-Type": contentType },
      conditions: [
        ["content-length-range", "1", "100"], // Content must be between 1 and 100 bytes
      ],
    });

    // First try with valid content size - should work
    const validFormData = new FormData();
    Object.entries(fields).forEach(([key, value]) => {
      validFormData.append(key, value);
    });
    validFormData.append("file", new Blob([smallContent], { type: contentType }));

    const validResponse = await fetch(url, {
      method: "POST",
      body: validFormData,
    });
    assertEquals(validResponse.status, 204);

    // Now try with content that violates the policy - should fail
    const invalidFormData = new FormData();
    Object.entries(fields).forEach(([key, value]) => {
      invalidFormData.append(key, value);
    });
    invalidFormData.append("file", new Blob([largeContent], { type: contentType }));

    const invalidResponse = await fetch(url, {
      method: "POST",
      body: invalidFormData,
    });

    // Should receive an error status code
    assert(invalidResponse.status >= 400);

    // Parse the error response XML to verify it's a policy violation
    const errorXml = await invalidResponse.text();
    assert(
      errorXml.includes("EntityTooLarge") || errorXml.includes("MaxSizeExceeded") ||
        errorXml.includes("ConditionFailed"),
    );

    // Clean up
    await client.deleteObject(objectName);
  },
});