adding next reformatting step

Author: djbard

README

@@ -79,4 +79,19 @@ cosmoflow-sims/pycola> python reformatNBody-noCiC-Ns-128from256.py  0
 
 The argument for this defines which file in the pycola output dir you start with. I put this in so I could run multiple versions of this code at the same time over the same output dir. 
 
-The output of this code is a directory inside the OmSiNs dir that has the name of the 3 cosmo params behind the simulation. Inside that dir are 8 npy files, each of which is a histogram of particle counts. 
+The output of this code is a directory inside the OmSiNs dir that has the name of the 3 cosmo params behind the simulation. Inside that dir are 8 npy files, each of which is a histogram of particle counts. 
+
+
+--------------------
+
+Making the list of files for input into the CosmoFlow IO code
+
+The next stage of formatting involves putting these sub-volume files into a format required by the CosmoFlow IO code, for when it makes the TFRecord files you'll actually use in the tensorflow training. For this we need a list of cosmologies, and we need to re-name all the dirs we just made to numbers that correspond to the position of this simulation on the list. 
+I know this is not a very sensible solution. 
+
+First run :
+cosmoflow-sims/pycola/OmSiNs/makeList.py
+to make the list, then run:
+
+cosmoflow-sims/pycola/OmSiNs/copyFiles.py
+to make new dirs with new names. Note that I'm copying them, rather than moving the dir, because I am cautious and didn't want to have to re-make all these files again. 

pycola/OmSiNs/copyFiles.py

@@ -0,0 +1,37 @@
+import os, shutil
+
+## need to rename all the files in the numbering scheme required by the CosmoFlow IO sceme. 
+## Previously, the dir had a useful name containing the comso params used in the sim
+## Now, this dir needs to be re-named so that it has a number, which corresponds to the position in the list of cosmo params
+## I know, this is not very sensible. But then the CosmoFlow IO code picks random sets of sims + sub-volumes to make the TFRecord files, and it's easier to do that when the file names are the numbers. 
+
+### In case you don't want to run this over all the files you have
+start = 1000
+stop = 1010
+
+for line in open("list.txt"):
+    if "#" in line:
+        continue
+    cols = line.split(",")
+    num = cols[0]
+    om = cols[1]
+    si = cols[2]
+    ns = cols[3][:-1]
+    print line, num, om, si, ns
+
+    if int(num)<start or int(num)>stop:
+        print"skipping!"
+        continue
+
+    outdir = "./"+num
+    if os.path.exists(outdir):
+        if len(os.listdir(outdir))==8:
+            print "done this one!"
+            continue
+        else:
+            shutil.rmtree(outdir)
+
+    ## now move the files 
+    indir = "pycola_"+om+"_"+si+"_"+ns
+    shutil.copytree(indir, outdir)
+    

pycola/OmSiNs/makeList.py

@@ -0,0 +1,29 @@
+import os, shutil
+
+### making the list of cosmological params, which will be used by the CosmoFlow IO code to make the tensorflow files
+
+list = open("list.txt", "w")
+print >> list, "## seed, om, si"
+counter = 999
+
+start = 0
+stop = 2000
+
+ct = 0
+for afile in os.listdir("./"):
+    if  "pycola" not in afile or "npz" in afile:
+        continue
+    else:
+        print afile
+        counter+=1
+        
+        if counter>3000:
+            continue
+            
+        om = afile.split("_")[1]
+        si = afile.split("_")[2]
+        ns = afile.split("_")[3]
+        print counter, om, si, ns
+        
+        print >> list,  str(counter)+","+str(om)+","+str(si)+","+str(ns)
+