Flattening Tuples, finding Variables by name, and converting Time Tuples

core/src/main/scala/latis/model/dataType.scala

@@ -5,17 +5,19 @@ import latis.metadata._
 import latis.util.DefaultDatumOrdering
 import latis.util.LatisException
 
+import scala.collection.mutable.ArrayBuffer
+
 /**
  * Define the algebraic data type for the LaTiS implementation of the
  * Functional Data Model.
  */
 sealed trait DataType extends MetadataLike with Serializable {
 
-  // Apply f to Scalars only, for now
+  /** Recursively apply f to this DataType. */
   def map(f: DataType => DataType): DataType = this match {
     case s: Scalar => f(s)
-    case Tuple(es @ _*) => Tuple(es.map(f))
-    case Function(d, r) => Function(d.map(f), r.map(f))
+    case t @ Tuple(es @ _*) => f(Tuple(t.metadata, es.map(f)))
+    case func @ Function(d, r) => f(Function(func.metadata, d.map(f), r.map(f)))
   }
 
   /**
@@ -44,6 +46,31 @@ sealed trait DataType extends MetadataLike with Serializable {
   def findVariable(variableName: String): Option[DataType] =
     getVariable(variableName)
     //TODO: support aliases
+    //TODO: could do findAllVariables(variableName).head like in v2
+
+  /**
+   * Find all Variables within this Variable by the given name.
+   * TODO: support aliases
+   */
+  def findAllVariables(variableName: String): Seq[DataType] = {
+    variableName.split('.') match {
+      case Array(_) =>
+        val vbuf = ArrayBuffer[DataType]()
+        if (this.id == variableName) vbuf += this //TODO: use hasName to cover aliases?
+        this match {
+          case _: Scalar =>
+          case Tuple(es @ _*) =>
+            vbuf ++= es.flatMap(_.findAllVariables(variableName))
+          case Function(d, r) => 
+            vbuf ++= d.findAllVariables(variableName) 
+            vbuf ++= r.findAllVariables(variableName)
+        }
+        vbuf.toSeq
+      case Array(n1, n2 @ _*) => {
+        findAllVariables(n1).flatMap(_.findAllVariables(n2.mkString(".")))
+      }
+    }
+  }
 
   /**
    * Return the function arity of this DataType.
@@ -73,34 +100,45 @@ sealed trait DataType extends MetadataLike with Serializable {
    * Return this DataType with all nested Tuples flattened to a single Tuple.
    * A Scalar will remain a Scalar.
    * This form is consistent with Samples which don't preserve nested Functions.
+   * Flattened Tuples retain the ID of the outermost Tuple.
    */
   def flatten: DataType = {
+    var tupIds = ""
     // Recursive helper function that uses an accumulator (acc)
     // to accumulate types while in the context of a Tuple
     // while the recursion results build up the final type.
     def go(dt: DataType, acc: Seq[DataType]): Seq[DataType] = dt match {
-      case s: Scalar      => acc :+ s
-      case Tuple(es @ _*) => es.flatMap(e => acc ++ go(e, Seq()))
-      case Function(d, r) => acc :+ Function(d.flatten, r.flatten)
+      //prepend Tuple ID(s) with dot(s) and drop leading dot(s)
+      case s: Scalar          => acc :+ s.rename(s"$tupIds.${s.id}".replaceFirst("^\\.+", ""))
+      case Function(d, r)     => acc :+ Function(d.flatten, r.flatten)
+      case t @ Tuple(es @ _*) => if (tupIds.isEmpty && !t.id.isEmpty) tupIds = t.id else tupIds += s".${t.id}"
+        es.flatMap(e => acc ++ go(e, Seq()))
     }
 
     val types = go(this, Seq())
     types.length match {
       case 1 => types.head
-      case _ => Tuple(types)
+      case _ => 
+        if (tupIds.split('.').isEmpty) Tuple(types)
+        else Tuple(Metadata(tupIds.split('.').head), types)
     }
   }
 
   /**
    * Return the path within this DataType to a given variable ID
    * as a sequence of SamplePositions.
    * Note that length of the path reflects the number of nested Functions.
+   * When searching a Tuple's ID, the path to the first Scalar in the Tuple is returned.
    */
   def getPath(id: String): Option[SamplePath] = {
 
     // Recursive function to try paths until it finds a match
     def go(dt: DataType, id: String, currentPath: SamplePath): Option[SamplePath] =
-      if (id == dt.id) Some(currentPath) //found it  //TODO: use hasName to cover aliases?
+      //TODO: use hasName to cover aliases?
+      //searching fully qualified ID with namespace
+      if (id.contains('.') && dt.id == id)    Some(currentPath) //found it
+      //searching variable ID without namespace
+      else if (dt.id.split('.').contains(id)) Some(currentPath) //found it
       else
         dt match { //recurse
           case _: Scalar => None //dead end

core/src/main/scala/latis/ops/TimeTupleToTime.scala

@@ -0,0 +1,78 @@
+package latis.ops
+
+import cats.implicits._
+import latis.data._
+import latis.metadata.Metadata
+import latis.model._
+import latis.time.Time
+import latis.util.LatisException
+
+/**
+ * Defines an Operation that converts a Tuple of time values to a single time Scalar.
+ *
+ * @param name the name of the Tuple that stores the time values
+ */
+case class TimeTupleToTime(name: String = "time") extends MapOperation {
+
+  override def applyToModel(model: DataType): DataType = {
+    val timeTuple = model.findAllVariables(name) match {
+      case Nil               => throw new LatisException(s"Cannot find variable: $name")
+      case vs: Seq[DataType] => vs.head
+    }
+
+    val time: Scalar = timeTuple match {
+      case Tuple(es @ _*) =>
+        //build up format string
+        val format: String = es.toList.traverse(_("units"))
+          .fold(throw new LatisException("A time Tuple must have units defined for each element."))(_.mkString(" "))
+        //make the time Scalar
+        val metadata = Metadata("id" -> "time", "units" -> format, "type" -> "string")
+        Time(metadata)
+      case _ => throw new LatisException(s"Variable '$name' must be a Tuple.")
+    }
+
+    model.map {
+      case t: Tuple if t.id == name => time //TODO: support aliases with hasName?
+      case v => v
+    }
+  }
+
+  override def mapFunction(model: DataType): Sample => Sample = {
+    val timePos: SamplePosition = model.getPath(name) match {
+      case Some(List(sp)) => sp
+      case _ => throw new LatisException(s"Cannot find path to variable: $name")
+    }
+    val timeLen: Int = model.findAllVariables(name) match {
+      case Nil               => throw new LatisException(s"Cannot find variable: $name")
+      case vs: Seq[DataType] => vs.head match {
+        case t: Tuple => t.flatten match {
+          case tf: Tuple => tf.elements.length //TODO: is this "dimensionality"? Should it be a first class citizen?
+        }
+        case _ => throw new LatisException(s"Variable '$name' must be a Tuple.")
+      }
+    }
+
+    (sample: Sample) => sample match {
+      case Sample(dd, rd) =>
+        //extract text values and join with space
+        //TODO: join with delimiter, problem when we use regex?
+        timePos match {
+          case DomainPosition(n) =>
+            val domain = dd.slice(0, n) ++ Seq(time(n, timeLen, dd)) ++ dd.slice(n+timeLen, dd.length)
+            Sample(domain, rd)
+          case RangePosition(n) =>
+            val range = rd.slice(0, n) ++ Seq(time(n, timeLen, rd)) ++ rd.slice(n+timeLen, rd.length)
+            Sample(dd, range)
+        }
+    }
+  }
+
+  /** Helper function to slice a time Datum out of a time Tuple given its position and length. */
+  private def time(pos: Int, len: Int, data: Seq[Data]): Datum = {
+    val timeData = data.slice(pos, pos+len)
+    Data.StringValue(
+      timeData.map { case d: Datum => d.asString }.mkString(" ")
+    )
+  }
+
+}