Merge pull request #135 from andrew-saydjari/2024_03_12

2024 03 12

pipeline.jl

@@ -6,7 +6,7 @@ using InteractiveUtils; versioninfo()
 Pkg.activate("./"); Pkg.instantiate(); Pkg.precompile()
 t_now = now(); dt = Dates.canonicalize(Dates.CompoundPeriod(t_now-t_then)); println("Package activation took $dt"); t_then = t_now; flush(stdout)
 using BLISBLAS
-using Distributed, SlurmClusterManager, Suppressor, DataFrames
+using Distributed, SlurmClusterManager, Suppressor, DataFrames, Random
 addprocs(SlurmManager(),exeflags=["--project=./"])
 t_now = now(); dt = Dates.canonicalize(Dates.CompoundPeriod(t_now-t_then)); println("Worker allocation took $dt"); t_then = t_now; flush(stdout)
 println("Running Main on ", gethostname())
@@ -59,6 +59,7 @@ using LibGit2; git_branch, git_commit = initalize_git(src_dir); @passobj 1 worke
     svalMarg0 = -0//10:1//10:0//10;
 
     cache_dir = "../local_cache/"
+    out_dir="../outdir/"
     inject_cache_dir = prior_dir*"2024_03_11/inject_local_cache_15672only_295_real"
 
     # Prior Dictionary
@@ -469,7 +470,7 @@ end
 end
 
 @everywhere begin
-    function multi_spectra_batch(indsubset; out_dir="../outdir", ddstaronly=ddstaronly)
+    function multi_spectra_batch(indsubset; out_dir=out_dir, ddstaronly=ddstaronly)
         ### Set up
         out = []
         startind = indsubset[1][1]
@@ -571,6 +572,7 @@ end
                 (x->x[metai][10],                       "ingestBit"),
                 (x->x[metai][11],                       "flux"),
                 (x->x[metai][12],                       "fluxerr2"),
+                (x->Int.(x[metai][13]),                 "simplemsk"),
                 (x->adjfibindx,                         "adjfiberindx"),
 
                 (x->Float64.(x[RVind][1][1]),           "RV_pixoff_final"),
@@ -658,6 +660,7 @@ end
             end
         end
         GC.gc()
+        return 0
     end
 
     function extractor(x,elemap,elename,savename)
@@ -686,7 +689,9 @@ nwork = length(workers())
 println("Batches to Do: $lenargs, number of workers: $nwork")
 flush(stdout)
 
-@showprogress pmap(multi_spectra_batch,ittot)
+rng = MersenneTwister(2024)
+pout = @showprogress pmap(multi_spectra_batch,ittot,on_error=ex->2)
+serialize(out_dir*"pout_apMADGICS.jdat",pout)
 rmprocs(workers())
 
 t_now = now(); dt = Dates.canonicalize(Dates.CompoundPeriod(t_now-t0)); println("Total script runtime: $dt"); t_then = t_now; flush(stdout)

src/chi2Wrappers.jl

@@ -18,6 +18,14 @@ end
 function slicer(lindx::Nothing,rindx::Int,widx)
     return (rindx-widx):rindx
 end
+function clamp_range(x,lbnd,ubnd)
+    if isempty(x)
+        return x
+    else
+        minrng, maxrng = extrema(x)
+        return maximum([lbnd,minrng]):minimum([maxrng,ubnd])
+    end
+end
 
 # passing Vcomb_0 is new to some of the codes, we need to revisit having to pass these intups
 function chi2_wrapper2d(svals,intup;sigslice=4)
@@ -32,8 +40,9 @@ function chi2_wrapper2d(svals,intup;sigslice=4)
     lindx = findfirst(wave_obs .>= (new_center*10^(sval1*6e-6) - sigslice*sval2))
     rindx = findlast(wave_obs .<= (new_center*10^(sval1*6e-6) + sigslice*sval2))
     widx = round(Int,sigslice*sval2/0.22) # this is a hardcoded approximate pixel size in Angstroms
-    slrng = slicer(lindx,rindx,widx)
+    slrng0 = slicer(lindx,rindx,widx)
     pre_Vslice .= view(ShiftedArrays.circshift(view(V_new,:,:,sigindx,tval),(rval,0)),simplemsk,:)
+    slrng = clamp_range(slrng0,1,size(pre_Vslice,1))
     pre_Vslice .*= Dscale 
     return woodbury_update_inv_tst_sub(
         Ctotinv,

src/prior_build/build_starLines.jl

@@ -47,8 +47,8 @@ using LibGit2; git_branch, git_commit = initalize_git(src_dir); @passobj 1 worke
 
 @everywhere begin
     nsub_out = 50
-    # normPercent = 94 #nothing turns it off
-    normPercent = nothing #94 #nothing turns it off
+    normPercent = 94 #nothing turns it off
+    # normPercent = nothing #94 #nothing turns it off
 
     nsub_rnd1 = 60
     rnd1size = 800

src/prior_build/build_starLines_dd.jl

@@ -74,12 +74,12 @@ using LibGit2; git_branch, git_commit = initalize_git(src_dir); @passobj 1 worke
     # Prior Dictionary
     prior_dict = Dict{String,String}()
 
-    prior_dict["past_run"] = prior_dir*"2023_09_26/outdir_wu_th/apMADGICS_out.h5" # update for the new run
-    prior_dict["map2visit"] = prior_dir*"2024_02_21/summary/dr17_dr17_map2visit_1indx.h5"
-    prior_dict["map2star"] = prior_dir*"2024_02_21/summary/dr17_dr17_map2star_1indx.h5"
+    prior_dict["past_run"] = prior_dir*"2024_03_12/outdir_wu_th/apMADGICS_out.h5" # update for the new run
+    prior_dict["map2visit"] = prior_dir*"2024_03_05/outlists/summary/dr17_dr17_map2visit_1indx.h5"
+    prior_dict["map2star"] = prior_dir*"2024_03_05/outlists/summary/dr17_dr17_map2star_1indx.h5"
 
     prior_dict["starLines_LSF"] = prior_dir*"2024_02_21/apMADGICS.jl/src/prior_build/starLine_priors_norm94/APOGEE_stellar_kry_50_subpix_f"
-    prior_dict["out_dir"] = prior_dir*"2024_02_21/apMADGICS.jl/src/prior_build/starLine_priors_norm94_dd/"
+    prior_dict["out_dir"] = prior_dir*"2024_03_12/apMADGICS.jl/src/prior_build/starLine_priors_norm94_dd/"
 end
 
 @everywhere begin
@@ -95,16 +95,16 @@ end
     RV_pixoff_final = reader(prior_dict["past_run"],"RV_pixoff_final")
 
     keyval = "x_residuals_z_v0"
-    savename_sub = chop(savename,tail=3)*"_"*keyval*".h5"
+    savename_sub = chop(prior_dict["past_run"],tail=3)*"_"*keyval*".h5"
     f = h5open(savename_sub)
     keyvalres = "x_residuals_v0"
-    savename_sub_res = chop(savename,tail=3)*"_"*keyvalres*".h5"
+    savename_sub_res = chop(prior_dict["past_run"],tail=3)*"_"*keyvalres*".h5"
     g = h5open(savename_sub_res)
     keyvalref = "x_starContinuum_v0"
-    savename_sub_ref = chop(savename,tail=3)*"_"*keyvalref*".h5"
+    savename_sub_ref = chop(prior_dict["past_run"],tail=3)*"_"*keyvalref*".h5"
     h = h5open(savename_sub_ref)
     keyvallineCof = "x_starLineCof_v0"
-    savename_sub_starLineCof = chop(savename,tail=3)*"_"*keyvallineCof*".h5"
+    savename_sub_starLineCof = chop(prior_dict["past_run"],tail=3)*"_"*keyvallineCof*".h5"
     m = h5open(savename_sub_starLineCof)
 
     function finalize_xnorm(intup,RV_pixoff_final,pf,V_starbasis)
@@ -251,12 +251,13 @@ sfd_msk = (sfd_reddening.<sfd_cut)
 
 ## MADGICS Cuts
 avg_flux_conservation = reader(prior_dict["past_run"],"avg_flux_conservation")
+adjfiberindx_vec = reader(prior_dict["past_run"],"adjfiberindx")
 msk_flux_conserve = (avg_flux_conservation .< flux_conserve_cut);
 RV_minchi2_final = reader(prior_dict["past_run"],"RV_minchi2_final")
 snr_proxy = sqrt.(-RV_minchi2_final);
 msk_MADGICS_snr = (snr_proxy .> snr_proxy_cut); 
 
-clean_msk = (adjfiberindx.<=300) 
+clean_msk = (adjfiberindx_vec.<=300) 
 clean_msk .&= (apg_msk .& (SNR.>DRP_SNR_CUT)) # Cuts on ASPCAP/Upstream Processing/Targetting
 clean_msk .&= sfd_msk # SFD Mask (low-reddening)
 clean_msk .&= (msk_flux_conserve .& msk_MADGICS_snr) # Remove StarConts that failed to converge well and low SNR model detections
@@ -265,7 +266,7 @@ println("Clean APO Visits for DD Model Training: $(count(clean_msk)), $(100*coun
 clean_inds = findall(clean_msk);
 h5write(prior_dict["out_dir"]*"clean_inds.h5","clean_inds_apo",clean_inds)
 
-clean_msk = (adjfiberindx.>300) 
+clean_msk = (adjfiberindx_vec.>300) 
 clean_msk .&= (apg_msk .& (SNR.>DRP_SNR_CUT)) # Cuts on ASPCAP/Upstream Processing/Targetting
 clean_msk .&= sfd_msk # SFD Mask (low-reddening)
 clean_msk .&= (msk_flux_conserve .& msk_MADGICS_snr) # Remove StarConts that failed to converge well and low SNR model detections

src/prior_build/sample_Korg.jl

@@ -75,7 +75,8 @@ t_now = now(); dt = Dates.canonicalize(Dates.CompoundPeriod(t_now-t_then)); prin
 
 rng = MersenneTwister(368)
 # Draw from DR17 Distribution
-f = FITS("/uufs/chpc.utah.edu/common/home/sdss/dr17/apogee/spectro/aspcap/dr17/turbo20/allStar-dr17-turbo20.fits")
+# f = FITS("/uufs/chpc.utah.edu/common/home/sdss/dr17/apogee/spectro/aspcap/dr17/turbo20/allStar-dr17-turbo20.fits") somewhat of a bug to have used turbo20 in first release, just not as entirely self consistent as I would have liked
+f = FITS(summary_file_by_dr("dr17","dr17",17,"allStar"))
 TEFF = read(f[2],"TEFF")
 LOGG = read(f[2],"LOGG")
 M_H = read(f[2],"M_H")

src/prior_build/submit_priors_np.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #SBATCH --account=sdss-np
 #SBATCH --partition=sdss-shared-np
-#SBATCH --nodes=2
+#SBATCH --nodes=4
 #SBATCH --ntasks-per-node=64
 
 #SBATCH --mem=0 #requesting all of the memory on the node
@@ -32,9 +32,9 @@ echo $SLURM_JOB_NODELIST
 # julia +1.10.0 sample_Korg.jl # 966.4 core-h, 2.5h on 6 nodes, 34.8 core-s/spec, 100% cpu usage
 # julia +1.10.0 build_starLines.jl # 40 core-h, 40 min on 1 node, 50% cpu usage
 # ----- indep -----
-julia +1.10.0 build_DIB.jl # 145 core-h, 2.3 h on 1 node, 100% cpu usage
+# julia +1.10.0 build_DIB.jl # 145 core-h, 2.3 h on 1 node, 100% cpu usage
 # ----- second pass -----
-# julia +1.10.0 build_starLines_dd.jl # try on 3 nodes? new parallelism strategy to try to mitigate time
+julia +1.10.0 build_starLines_dd.jl # try on 4 nodes? new parallelism strategy to try to mitigate time
 
 
 # Clean up logs and Report Timing

submit_np.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #SBATCH --account=sdss-np
 #SBATCH --partition=sdss-shared-np
-#SBATCH --nodes=8
+#SBATCH --nodes=4
 #SBATCH --ntasks-per-node=64
 
 #SBATCH --mem=0 #requesting all of the memory on the node