RD-14694: Change Location.Ls to return a list of locations (#503)

snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/LocationPackageTest.scala

@@ -91,4 +91,18 @@ class LocationPackageTest extends SnapiTestContext with LocalLocationsTestContex
         |Error.Build("file system error: path not found: /not/found"))""".stripMargin
     )
   )
+
+  test(s"""
+    |let url = "$authorsJsonLocalDirectory",
+    |    people = List.Unnest(Location.Ls(url), loc -> Json.Read(loc, type list(record(name: string, title: string))))
+    |in List.Transform(List.GroupBy(people.title, job -> job), row -> {job: row.key, n: List.Count(row.group)})
+    |""".stripMargin) { it =>
+    it should evaluateTo("""[
+      | {job: "professor", n:18},
+      | {job: "engineer", n: 5},
+      | {job: "assistant professor", n: 11},
+      | {job: "PhD", n: 16}
+      |]""".stripMargin)
+  }
+
 }

snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/LocationPackageTest.scala

@@ -39,7 +39,7 @@ class LocationPackageTest extends SnapiTestContext {
     |  Collection.Count(data)
     |""".stripMargin)(it => it should evaluateTo("7"))
 
-  test(s"""Location.Ls("s3://$UnitTestPublicBucket/publications/")""") { it =>
+  test(s"""List.Transform(Location.Ls("s3://$UnitTestPublicBucket/publications/"), x -> String.From(x))""") { it =>
     it should evaluateTo("""Collection.Build(
       |  "s3://rawlabs-public-test-data/publications/authors.parquet",
       |  "s3://rawlabs-public-test-data/publications/authors.hjson",
@@ -99,4 +99,16 @@ class LocationPackageTest extends SnapiTestContext {
   test(s"""String.Read(S3.Build("$UnitTestPrivateBucket2", "/file1.csv"))
     |""".stripMargin)(it => it should evaluateTo(""" "foobar" """))
 
+  test(s"""let dir = S3.Build(
+    |      "$UnitTestPrivateBucket", "/publications/publications-hjson/*.json",
+    |      region = "${UnitTestPrivateBucketCred.getRegion}",
+    |      accessKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getAccessKey}",
+    |      secretKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getSecretKey}"
+    |    ),
+    |    files = Location.Ls(dir),
+    |    lines = List.Unnest(files, f -> List.From(String.ReadLines(f)))
+    |in
+    |  List.Count(lines)
+    |""".stripMargin)(it => it should evaluateTo("1000"))
+
 }

snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/S3PackageTest.scala

@@ -42,7 +42,7 @@ class S3PackageTest extends SnapiTestContext {
 
   // listing a s3 bucket from us-east-1 (non default region)
   test(s"""let
-    |  data = Location.Ls(
+    |  data = Location.Ll(
     |    S3.Build(
     |      "$unitTestPrivateBucketUsEast1",
     |      "/csvs/01",
@@ -52,15 +52,15 @@ class S3PackageTest extends SnapiTestContext {
     |    )
     |  )
     |in
-    |  data
+    |  data.url
     |""".stripMargin)(it => it should evaluateTo("""[
     |   "s3://rawlabs-unit-tests-us-east-1/csvs/01/data2.csv",
     |   "s3://rawlabs-unit-tests-us-east-1/csvs/01/data1.csv"
     |]""".stripMargin))
 
   // listing a s3 bucket from us-east-1 without passing the region
   test(s"""let
-    |  data = Location.Ls(
+    |  data = Location.Ll(
     |    S3.Build(
     |      "$unitTestPrivateBucketUsEast1",
     |      "/csvs/01",
@@ -69,7 +69,7 @@ class S3PackageTest extends SnapiTestContext {
     |    )
     |  )
     |in
-    |  data
+    |  data.url
     |""".stripMargin)(it => it should evaluateTo("""[
     |   "s3://rawlabs-unit-tests-us-east-1/csvs/01/data2.csv",
     |   "s3://rawlabs-unit-tests-us-east-1/csvs/01/data1.csv"

snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/regressions/credentials/RD4445Test.scala

@@ -27,12 +27,12 @@ class RD4445Test extends SnapiTestContext {
     _ should evaluateTo("""["Hello", "World", "Again!"]""")
   )
 
-  test("""Location.Ls("dropbox://rawlabs-dropbox/New Folder")""")(
+  test("""Location.Ll("dropbox://rawlabs-dropbox/New Folder").url""")(
     _ should evaluateTo("""["dropbox:/New Folder/New Document"]""")
   )
 
   // Listing same folder but with trailing '/'
-  test("""Location.Ls("dropbox://rawlabs-dropbox/New Folder/")""")(
+  test("""Location.Ll("dropbox://rawlabs-dropbox/New Folder/").url""")(
     _ should evaluateTo("""["dropbox:/New Folder/New Document"]""")
   )
 

snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/builtin/LocationPackage.scala

@@ -190,11 +190,11 @@ class LocationLsEntry extends EntryExtension {
 
   override def entryName: String = "Ls"
 
-  override def docs = EntryDoc(
+  override def docs: EntryDoc = EntryDoc(
     summary = "Lists a location. The location must be a file system or an S3 bucket.",
     params = List(ParamDoc("value", TypeDoc(List("location")), "The location to list.")),
     examples = List(ExampleDoc("""Location.Ls("s3://my-bucket/folder/")""", None)),
-    ret = Some(ReturnDoc("The list of files in the location.", retType = Some(TypeDoc(List("list(string)"))))),
+    ret = Some(ReturnDoc("The list of files in the location.", retType = Some(TypeDoc(List("list(location)"))))),
     description = Some(
       "Urls with wildcards are also supported. For information about the use of wildcards see the [Locations with wildcards documentation](/snapi/wildcards)."
     )
@@ -211,7 +211,7 @@ class LocationLsEntry extends EntryExtension {
       optionalArgs: Seq[(String, Arg)],
       varArgs: Seq[Arg]
   )(implicit programContext: ProgramContext): Either[String, Type] =
-    Right(SnapiListType(SnapiStringType(), Set(SnapiIsTryableTypeProperty())))
+    Right(SnapiListType(SnapiLocationType(), Set(SnapiIsTryableTypeProperty())))
 
 }
 

snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationLsNode.java

@@ -18,31 +18,33 @@
 import com.oracle.truffle.api.nodes.NodeInfo;
 import com.rawlabs.snapi.frontend.snapi.extensions.LocationDescription$;
 import com.rawlabs.snapi.truffle.ast.ExpressionNode;
-import com.rawlabs.snapi.truffle.runtime.list.StringList;
+import com.rawlabs.snapi.truffle.runtime.list.ObjectList;
 import com.rawlabs.snapi.truffle.runtime.primitives.ErrorObject;
 import com.rawlabs.snapi.truffle.runtime.primitives.LocationObject;
 import com.rawlabs.utils.core.RawException;
 import com.rawlabs.utils.sources.filesystem.api.FileSystemLocation;
 import scala.collection.IndexedSeq;
 
-@NodeInfo(shortName = "String.Read")
+@NodeInfo(shortName = "Location.Ls")
 @NodeChild("location")
 public abstract class LocationLsNode extends ExpressionNode {
   @Specialization
   @TruffleBoundary
   protected Object doLs(LocationObject locationObject) {
     try {
       FileSystemLocation fs = locationObject.getFileSystemLocation();
-      IndexedSeq<String> values =
-          fs.ls().map(LocationDescription$.MODULE$::locationToPublicUrl).toIndexedSeq();
+      IndexedSeq<LocationObject> values =
+          fs.ls()
+              .map(l -> new LocationObject(l, LocationDescription$.MODULE$.locationToPublicUrl(l)))
+              .toIndexedSeq();
       int size = values.size();
-      String[] result = new String[size];
+      Object[] result = new LocationObject[size];
 
       for (int i = 0; i < size; i++) {
         result[i] = values.apply(i);
       }
 
-      return new StringList(result);
+      return new ObjectList(result);
     } catch (RawException e) {
       return new ErrorObject(e.getMessage());
     }