Как скопировать Julia DataFrame в поток l oop? - PullRequest
Новая
       84

Как скопировать Julia DataFrame в поток l oop?

1 голос
/ 02 августа 2020

У меня следующая проблема:

DataFrame с именем cr1 с 553 столбцами

Затем я делаю два цикла следующим образом: 

cr2=copy(cr1)

   #-----------------#
   #Loop on scenarios#
   #-----------------#
    @threads for k in 0:499
        cr1=copy(cr2)
        cr1[!,"scn"].=k

        cr1 = leftjoin(cr1,tra,on = [:scn])

        for i in 1:40
            #----------------------#
            ##Projection on 40 years
            #----------------------#
            cr1[!,"ch_prime"*str(i)].=cr1[!,"ch_prime"].*(cr1[!,"coll_"*str(i)]
            ...job on other columns

На Вначале я делаю копию cr1, чтобы сохранить видение DataFrame на этом этапе. Действительно, мне нужно перезапускать это видение на каждой итерации kl oop. Затем я рассматриваю сценарий = k для правильного соединения с DF, называемым tra, с 40 столбцами (от tra1 до tra40) и k строками. Наконец, я делаю еще l oop, чтобы спроецировать все свои переменные на 40 лет.

Но это не работает. Джулия делает некоторую работу, но cr1 не изменяется.

Я попытался добавить глобал в первую строку l oop и сжать @ thread 

for k in 0:499
        global cr1=copy(cr2)

Тогда это работает, но без многопоточности, и я хочу сохранить его для повышения производительности

Есть ли у вас какой-нибудь другой способ сделать это или способ перенести последнее решение?

EDIT

Вот полный код (извините, немного длинный), если он может помочь: 

#Définit la fonction string avec une syntaxe plus courte
#Permet de conserver le code Python
function str(x)
    string(x)
end


# %% Début du programme
@time begin
    ##+ Import des tables
    donnee=CSV.File("/Users/Python/Python_VIF_sto/donnee.csv")|> DataFrame!
    lapse=CSV.File("/Users/Python/Python_VIF_sto/lapse.csv")|> DataFrame!
    dc=CSV.File("/Users/Python/Python_VIF_sto/dc.csv")|> DataFrame!
    ind_action=CSV.File("/Users/Python/Python_VIF_sto/ind_action.csv")|> DataFrame!
    tra=CSV.File("/Users/Python/Python_VIF_sto/tra.csv")|> DataFrame!
    ts=CSV.File("/Users/Python/Python_VIF_sto/ts.csv")|> DataFrame!
    ech=CSV.File("/Users/Python/Python_VIF_sto/ech.csv")|> DataFrame!
    sw=CSV.File("/Users/Python/Python_VIF_sto/sw.csv")|> DataFrame!
    df=CSV.File("/Users/Python/Python_VIF_sto/df.csv")|> DataFrame!
    tx_is=CSV.File("/Users/Python/Python_VIF_sto/tx_is.csv")|> DataFrame!
    fga=CSV.File("/Users/Python/Python_VIF_sto/fga.csv")|> DataFrame!
    fgp=CSV.File("/Users/Python/Python_VIF_sto/fgp.csv")|> DataFrame!
    fgacq=CSV.File("/Users/Python/Python_VIF_sto/fgacq.csv")|> DataFrame!
    inf=CSV.File("/Users/Python/Python_VIF_sto/inf.csv")|> DataFrame!

    #Definition des parametres communs
    ppb0=1347911164;     # PPB initiale
    const global dur_proj=40;         # Durée de projection
    nb_coll=6;           # 6 années de collecte possible
    tcomp_mod=0         # Mode de determination composante technique PB min (1=fixe ou 0=libre)
    tcomp_fixe=1        # Composante si fixe (1=prev 0=RT)
    fdc=1               # 1 = frontiere des contrats S2 / 0 = désactivée

    #CR initial avec les paramètres en t=0#
    cr1=copy(donnee)
    cr1[!, :annee].= 1
    cr1[!, :der_fg].=0.01  #Dérive des frais au delà inflation


    #Creation des dummies#
    cr1[!,:eu] .= (cr1.supp.=="eu")
    cr1[!,:uc] .= (cr1.supp.=="uc")
    cr1[!, :prev] .= (cr1.supp.=="prev")
    cr1[!, :ve] .= (cr1.supp.=="ve")


    #Construction des taux projetés sur horizon de projection(rachats, décès, indices..)#

    #Deces - par produit#
    cr1 = leftjoin(cr1,dc, on =[:age, :tab_dc,:ab_dc]) # Décès
    #----------------------#
    #Echeances & Arbitrages#
    #----------------------#

    cr1 = leftjoin(cr1,ech, on =[:tab_ech])   #Echeances
    cr1 = leftjoin(cr1,sw, on =[:tab_sw])     #Arbitrages

    #---------------------------#
    #Frais admin / acq / prestas#
    #---------------------------#
    cr1 = leftjoin(cr1,fga, on =[:fg_ad])     #Frais d"administration
    cr1 = leftjoin(cr1,fgp, on =[:fg_prest])  #Frais sur prestations
    cr1 = leftjoin(cr1,fgacq,on =[:fg_acq])  #Frais d"acquisition

    #------------#
    #---Lapses---#
    #------------#
    cr1 = leftjoin(cr1,lapse,on = [:tab_lapse]) #Rachats - vision simple - pas de gestion de la collecte

    m = Matrix(cr1[!,columnindex(cr1, :anc1):columnindex(cr1, :anc100)])
    v = cr1.anc
    insertcols!(cr1, ["lap$k" => getindex.(Ref(m), axes(cr1, 1), v .+ k .- 1) for k in 1:dur_proj]...)

    #--------#
    #Collecte#
    #--------#
    # On force la collecte à zero au dela du plan
    for i in 7:dur_proj
        cr1[!,"coll_"*str(i)].=0
    end

    # Collecte prevoyance
    for i in 1:dur_proj
        cr1[!,"ca_prev"*str(i)].=cr1[!,"ca_prev"*str(i-1)].*(1 .-cr1[!,"lap"*str(i)])
        .+cr1[!,"coll_"*str(i)].*cr1[!,"prev"]
    end

    # Application de la frontière des contrats au delà de la première année
    for i in 2:dur_proj
        cr1[!,"ca_prev"*str(i)].=cr1[!,"ca_prev"*str(i)].*cr1[!,"fdc"]
    end

    #------------------------------------------#
    #Allocation PPB ini aux prorata des PM Euro#
    #------------------------------------------#

    s=(cr1[!,"eu"].+cr1[!,"ve"]).*cr1[!,"pm_0"]
    som=max(1,sum(s))                                                                      #Eviter les divisions par zero qd pas de produits euros
    ratio=s/som
    cr1[!,"ppb0"].=ppb0.*ratio                                                                  #Quid de la prev


    cr2=copy(cr1)

   #------------------------#
    #Boucle sur les scenarios#
    #------------------------#
    @threads for k in 0:499
        global cr1=copy(cr2)
        cr1[!,"scn"].=k   #scenario deterministe / parametrable en sto

        #---------------------------------#
        #Paramètres d"actifs et financiers#
        #---------------------------------#
        cr1 = leftjoin(cr1,tra,on = [:scn])           #TRA
        cr1 = leftjoin(cr1,ts,on = [:tab_ts])         #TS
        cr1 = leftjoin(cr1,ind_action,on = [:scn])    #Indice actions
        cr1 = leftjoin(cr1,df,on = [:scn])            #Discount Factor
        cr1 = leftjoin(cr1,tx_is,on = [:scn])         #Taux d"IS
        cr1 = leftjoin(cr1,inf,on = [:scn])           #Inflation


        ##CR deterministe sur duree de projection
        for i in 1:dur_proj
            #---------#
            ##Technique
            #---------#
            cr1[!,"ch_prime"*str(i)].=cr1[!,"ch_prime"].*(cr1[!,"coll_"*str(i)].*(cr1[!,"eu"]
            .+cr1[!,"uc"].+cr1[!,"ve"]).+cr1[!,"ca_prev"*str(i)])##chargements sur primes OK
            cr1[!,"dc"*str(i)].=cr1[!,"qx"*str(i)].*(cr1[!,"pm_"*str(i-1)].+cr1[!,"ca_prev"*str(i)])#deces OK
            cr1[!,"rach"*str(i)].=cr1[!,"lap"*str(i)].*cr1[!,"pm_"*str(i-1)].*(cr1[!,"eu"].+cr1[!,"uc"].+cr1[!,"ve"]) #Rachats structurels KO - Defo ancienneté avec collecte
            cr1[!,"eche"*str(i)].=cr1[!,"ech"*str(i)].*cr1[!,"pm_"*str(i-1)]#Echeances OK
            #cr1["arb"*str(i)]=cr1["sw"*str(i)]*cr1["pm_"+str(i-1)] #Arbitrages € vers UC KO lien produit multi
            cr1[!,"ch_enc"*str(i)].=cr1[!,"ch_enc"].*cr1[!,"pm_"*str(i-1)].+(cr1[!,"coll_"*str(i)]
            .-cr1[!,"rach"*str(i)].-cr1[!,"dc"*str(i)]).*((1 .+cr1[!,"ch_enc"]).^(0.5).- 1)#chargements sur encours OK
            #RT Prevoyance (intègre de fait les chargements sur primes => à oter du résultat ass)
            cr1[!,"rt"*str(i)].=(1 .-cr1[!,"SP"].-cr1[!,"ch_prime"]).*cr1[!,"ca_prev"*str(i)].*cr1[!,"prev"]

            #---------#
            ##Financier
            #---------#
            #IT#
            cr1[!,"IT"*str(i)].=(cr1[!,"eu"].+cr1[!,"ve"]).*(cr1[!,"IT"].*cr1[!,"pm_"*str(i-1)].+(cr1[!,"coll_"*str(i)]
            .-cr1[!,"rach"*str(i)].-cr1[!,"dc"*str(i)]).*((1 .+cr1[!,"IT"]).^(0.5).-1))#IT OK
            #PB distribuée
            cr1[!,"pb_dist"*str(i)].=(cr1[!,"eu"].+cr1[!,"ve"]).*((cr1[!,"ts"*str(i)].-cr1[!,"IT"]).*cr1[!,"pm_"*str(i-1)]
            .+(cr1[!,"coll_"*str(i)].-cr1[!,"rach"*str(i)].-cr1[!,"dc"*str(i)]).*((1 .+cr1[!,"ts"*str(i)]
            .-cr1[!,"IT"]).^(0.5).-1))#PB dist OK
            #Ajustements ACAV
            cr1[!,"acav"*str(i)].=(cr1[!,"uc"].*cr1[!,"pm_"*str(i-1)].*(cr1[!,"inda"*str(i+1)]./cr1[!,"inda"*str(i)].-1)
            .+(cr1[!,"coll_"*str(i)].-cr1[!,"ch_prime"*str(i)].-cr1[!,"rach"*str(i)].-cr1[!,"dc"*str(i)]).*cr1[!,"uc"]
            .*(cr1[!,"inda"*str(i+1)]./cr1[!,"inda"*str(i)].-1)./2)

            #Produits financiers contrats
            cr1[!,"pfi"*str(i)].=(cr1[!,"eu"].+cr1[!,"ve"]).*cr1[!,"tra"*str(i)].*(2 .*cr1[!,"pm_"*str(i-1)]
            .+cr1[!,"coll_"*str(i)].-cr1[!,"rach"*str(i)].-cr1[!,"dc"*str(i)])./2 #produits financiers par produits - hors PPB - hors FP

            #Produits financiers sur PPB <=> Au prorata pfi contrats euros(suffisant en vision marginale)
            cr1[!,"pfi_ppb"*str(i)].=cr1[!,"ppb"*str(i-1)].*cr1[!,"tra"*str(i)]

            #CSG
            cr1[!,"csg"*str(i)].=(cr1[!,"eu"].+cr1[!,"ve"]).*cr1[!,"csg"].*(cr1[!,"pb_dist"*str(i)].+cr1[!,"IT"*str(i)].-cr1[!,"ch_enc"*str(i)])

            ##PM fin
            cr1[!,"pm_"*str(i)].=(cr1[!,"pm_"*str(i-1)].+cr1[!,"coll_"*str(i)].+cr1[!,"ca_prev"*str(i)].-cr1[!,"ch_prime"*str(i)]
            .-cr1[!,"ch_enc"*str(i)].-cr1[!,"dc"*str(i)].-cr1[!,"rach"*str(i)].+cr1[!,"acav"*str(i)].-cr1[!,"eche"*str(i)]
            .+cr1[!,"IT"*str(i)].+cr1[!,"pb_dist"*str(i)].-cr1[!,"csg"*str(i)]).*(cr1[!,"eu"].+cr1[!,"ve"].+cr1[!,"uc"])

            #Retrocessions UC
            cr1[!,"retro"*str(i)].=cr1[!,"retro"].*(cr1[!,"pm_"*str(i-1)].+cr1[!,"pm_"*str(i)]).*0.5

            #Correctifs ratios de frais
            cr1[!,"cor_fg_uc0"].=1
            cr1[!,"cor_fg_eu0"].=1

            cr1[!,"cor_fg_uc"*str(i)].=cr1[!,"cor_fg_uc"*str(i-1)].*((cr1[!,"inf"*str(i-1)].+cr1[!,"der_fg"])
            ./(cr1[!,"inda"*str(i+1)]./cr1[!,"inda"*str(i)]))
            cr1[!,"cor_fg_eu"*str(i)].=cr1[!,"cor_fg_eu"*str(i-1)].*((cr1[!,"inf"*str(i-1)].+cr1[!,"der_fg"])
            ./(1 .+cr1[!,"ts"*str(i)].+cr1[!,"IT"]))

            #Frais généraux
            ##FG d"administration epargne
            cr1[!,"fg_ad"*str(i)].=cr1[!,"fga"*str(i)].*(cr1[!,"cor_fg_uc"*str(i)].*cr1[!,"uc"].+cr1[!,"cor_fg_eu"*str(i)]
            .*(cr1[!,"eu"].+cr1[!,"prev"].+cr1[!,"ve"])).*(cr1[!,"pm_"*str(i-1)].+cr1[!,"pm_"*str(i)].-(cr1[!,"IT"*str(i)]
            .+cr1[!,"pb_dist"*str(i)].+cr1[!,"csg"*str(i)]).*(cr1[!,"eu"].+cr1[!,"ve"]))./2
            ##FG d"aquisition
            cr1[!,"fg_acq"*str(i)].=cr1[!,"fgacq"*str(i)].*(cr1[!,"coll_"*str(i)].+cr1[!,"ca_prev"*str(i)])
            #FG Prestations
            cr1[!,"fg_prest"*str(i)].=cr1[!,"fgp"*str(i)].*(cr1[!,"cor_fg_uc"*str(i)].*cr1[!,"uc"].+cr1[!,"cor_fg_eu"*str(i)]
            .*(cr1[!,"eu"].+cr1[!,"prev"].+cr1[!,"ve"])).*(cr1[!,"rach"*str(i)].+cr1[!,"dc"*str(i)].+cr1[!,"ca_prev"*str(i)]
            .*cr1[!,"SP"])

            #Coms de gestion & admin (mma)
            cr1[!,"com_ad"*str(i)].=cr1[!,"com_ad"].*(cr1[!,"pm_"*str(i-1)].+cr1[!,"pm_"*str(i)].-(cr1[!,"IT"*str(i)]
            .+cr1[!,"pb_dist"*str(i)].+cr1[!,"csg"*str(i)]).*(cr1[!,"eu"].+cr1[!,"ve"]))./2#OK
            cr1[!,"com_acq"*str(i)].=cr1[!,"com_acq"].*cr1[!,"coll_"*str(i)]##=>OK

            #PB min
            #Solde souscription
            cr1[!,"ssous"*str(i)].=(cr1[!,"eu"].+cr1[!,"ve"].+cr1[!,"prev"]).*(cr1[!,"ch_enc"*str(i)].+cr1[!,"rt"*str(i)]
            .+cr1[!,"ch_prime"*str(i)].-cr1[!,"fg_ad"*str(i)].-cr1[!,"fg_acq"*str(i)].-cr1[!,"fg_prest"*str(i)]
            .-cr1[!,"com_ad"*str(i)].-cr1[!,"com_acq"*str(i)])

            #Determination composante technique (4,5% Pprev ou 10% tech)
            if tcomp_mod==1
                cr1[!,"tcomp"*str(i)].=tcomp_fixe
            elseif tcomp_mod==0 && sum(cr1[!,"ca_prev"*str(i)]).*0.045 .> sum(cr1[!,"ssous"*str(i)]).*0.1
                cr1[!,"tcomp"*str(i)].=1
            else
                cr1[!,"tcomp"*str(i)].=0
            end

            #Quid du changement de contrainte sur le technique en fonction des produits enlevés?
            cr1[!,"pb_min"*str(i)].=0.85.*(cr1[!,"pfi"*str(i)].+cr1[!,"pfi_ppb"*str(i)]).+(cr1[!,"ssous"*str(i)]
            .-cr1[!,"tcomp"*str(i)].*cr1[!,"ca_prev"*str(i)].*0.045) .-((1 .-cr1[!,"tcomp"*str(i)]).*cr1[!,"ssous"*str(i)].*0.10)

            #Variation de PPB par produit
            cr1[!,"varppb"*str(i)].=cr1[!,"pb_min"*str(i)].-cr1[!,"IT"*str(i)].-cr1[!,"pb_dist"*str(i)]

            #PPB par produit
            cr1[!,"ppb"*str(i)].=cr1[!,"ppb"*str(i-1)].+cr1[!,"varppb"*str(i)]

            #RT Vie entière
            cr1[!,"rtve"*str(i)].=cr1[!,"ve"].*(-1).*(cr1[!,"ch_enc"*str(i)].-cr1[!,"fg_ad"*str(i)].-cr1[!,"fg_prest"*str(i)]
            .-(cr1[!,"ca_prev"*str(i)].-cr1[!,"ch_prime"*str(i)].-cr1[!,"varppb"*str(i)].-cr1[!,"fg_ad"*str(i)]
            .-cr1[!,"fg_prest"*str(i)].-cr1[!,"dc"*str(i)].-cr1[!,"rach"*str(i)]))

            #PM finale yc RT Vie entière
            cr1[!,"pm_"*str(i)].=cr1[!,"pm_"*str(i)].+cr1[!,"rtve"*str(i)].-cr1[!,"varppb"*str(i)].*cr1[!,"ve"]

            #Impots
            cr1[!,"IS"*str(i)].=(cr1[!,"ch_prime"*str(i)].+cr1[!,"ch_enc"*str(i)].+cr1[!,"pfi"*str(i)].+cr1[!,"pfi_ppb"*str(i)]
            .-cr1[!,"pb_min"*str(i)].+cr1[!,"retro"*str(i)].+cr1[!,"rt"*str(i)].-cr1[!,"fg_ad"*str(i)].-cr1[!,"fg_acq"*str(i)]
            .-cr1[!,"fg_prest"*str(i)].-cr1[!,"com_ad"*str(i)].-cr1[!,"com_acq"*str(i)]).*cr1[!,"tx_is"*str(i)]

            #Résultat net Assurance apres IS
            cr1[!,"rnai"*str(i)].=(cr1[!,"ch_prime"*str(i)].+cr1[!,"ch_enc"*str(i)].+cr1[!,"pfi"*str(i)].+cr1[!,"pfi_ppb"*str(i)]
            .-cr1[!,"pb_min"*str(i)].+cr1[!,"retro"*str(i)].+cr1[!,"rt"*str(i)].-cr1[!,"fg_ad"*str(i)].-cr1[!,"fg_acq"*str(i)]
            .-cr1[!,"fg_prest"*str(i)].-cr1[!,"com_ad"*str(i)].-cr1[!,"com_acq"*str(i)]).*(1 .-cr1[!,"tx_is"*str(i)])

            #Profits futurs "actualisés (PVFP)
            cr1[!,"pvfp"*str(i)].=cr1[!,"rnai"*str(i)].*cr1[!,"df"*str(i)]
        end


        #---------------------------------------------#
        #Variables d"intérêt actualisés sur duree proj
        #Principaux éléments de marge
        #---------------------------------------------#
        cr1[!,"ch_enc_actu"].=0
        cr1[!,"ch_prime_actu"].=0
        cr1[!,"pfnd_actu"].=0
        cr1[!,"retro_actu"].=0
        cr1[!,"rt_actu"].=0
        cr1[!,"fg_ad_actu"].=0
        cr1[!,"fg_acq_actu"].=0
        cr1[!,"fg_prest_actu"].=0
        cr1[!,"com_ad_actu"].=0
        cr1[!,"com_acq_actu"].=0
        cr1[!,"IS_actu"].=0
        cr1[!,"pvfp_actu"].=0
        cr1[!,"pm_actu"].=(cr1[!,"pm_0"].*cr1[!,"df1"].-cr1[!,"pm_"*str(dur_proj-2)].*cr1[!,"df"*str(dur_proj-1)])./2 #initialise
        cr1[!,"ca_actu"].=0
        cr1[!,"pfi_actu"].=0
        cr1[!,"pfi_ppb_actu"].=0
        cr1[!,"pb_dist_actu"].=0
        cr1[!,"IT_actu"].=0
        cr1[!,"varppb_actu"].=0
        cr1[!,"dc_actu"].=0
        cr1[!,"rach_actu"].=0
        cr1[!,"marge_sous_actu"].=0
        cr1[!,"marge_acq_actu"].=0
        cr1[!,"marge_ges_actu"].=0
        cr1[!,"marge_fi_actu"].=0


        for i in 1:dur_proj
           cr1[!,"ch_enc_actu"].=cr1[!,"ch_enc_actu"].+(cr1[!,"ch_enc"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"ch_prime_actu"].=cr1[!,"ch_prime_actu"].+(cr1[!,"ch_prime"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"pfnd_actu"].=cr1[!,"pfnd_actu"].+(cr1[!,"pfi"*str(i)].+cr1[!,"pfi_ppb"*str(i)].-cr1[!,"pb_min"*str(i)]).*cr1[!,"eu"].*cr1[!,"df"*str(i)]
           cr1[!,"retro_actu"].=cr1[!,"retro_actu"].+cr1[!,"retro"*str(i)].*cr1[!,"df"*str(i)]
           cr1[!,"rt_actu"].=cr1[!,"rt_actu"].+cr1[!,"rt"*str(i)].*cr1[!,"df"*str(i)]
           cr1[!,"fg_ad_actu"].=cr1[!,"fg_ad_actu"].+cr1[!,"fg_ad"*str(i)].*cr1[!,"df"*str(i)]
           cr1[!,"fg_acq_actu"].=cr1[!,"fg_acq_actu"].+cr1[!,"fg_acq"*str(i)].*cr1[!,"df"*str(i)]
           cr1[!,"fg_prest_actu"].=cr1[!,"fg_prest_actu"].+cr1[!,"fg_prest"*str(i)].*cr1[!,"df"*str(i)]
           cr1[!,"com_ad_actu"].=cr1[!,"com_ad_actu"].+cr1[!,"com_ad"*str(i)].*cr1[!,"df"*str(i)]
           cr1[!,"com_acq_actu"].=cr1[!,"com_acq_actu"].+cr1[!,"com_acq"*str(i)].*cr1[!,"df"*str(i)]
           cr1[!,"IS_actu"].=cr1[!,"IS_actu"].+(cr1[!,"IS"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"pvfp_actu"].=cr1[!,"pvfp_actu"].+cr1[!,"pvfp"*str(i)]#déjà actualisée
           cr1[!,"ca_actu"].=cr1[!,"ca_actu"].+(cr1[!,"ca_prev"*str(i)].+cr1[!,"coll_"*str(i)]).*cr1[!,"df"*str(i)]
           cr1[!,"pfi_actu"].=cr1[!,"pfi_actu"].+(cr1[!,"pfi"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"pfi_ppb_actu"].=cr1[!,"pfi_ppb_actu"].+(cr1[!,"pfi_ppb"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"pb_dist_actu"].=cr1[!,"pb_dist_actu"].+(cr1[!,"pb_dist"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"IT_actu"].=cr1[!,"IT_actu"].+(cr1[!,"IT"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"varppb_actu"].=cr1[!,"varppb_actu"].+(cr1[!,"varppb"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"dc_actu"].=cr1[!,"dc_actu"].+(cr1[!,"dc"*str(i)].*cr1[!,"df"*str(i)])
           cr1[!,"rach_actu"].=cr1[!,"rach_actu"].+(cr1[!,"rach"*str(i)].*cr1[!,"df"*str(i)])
        end
        for i in 1:dur_proj-1
           cr1[!,"pm_actu"].=cr1[!,"pm_actu"].+(cr1[!,"pm_"*str(i)].*cr1[!,"df"*str(i+1)])
        end
        cr1[!,"marge_sous_actu"].=cr1[!,"rt_actu"]
        cr1[!,"marge_acq_actu"].=cr1[!,"ch_prime_actu"].-cr1[!,"com_acq_actu"].-cr1[!,"fg_acq_actu"]
        cr1[!,"marge_ges_actu"].=cr1[!,"ch_enc_actu"].+cr1[!,"retro_actu"].-cr1[!,"com_ad_actu"].-cr1[!,"fg_ad_actu"].-cr1[!,"fg_prest_actu"]
        cr1[!,"marge_fi_actu"].=cr1[!,"pfnd_actu"]

#        cr3 = cr1[:, All(Between(:gamme,:pm_0), Between(:ch_enc_actu,:marge_fi_actu))]

    end
end

1 Ответ

0 голосов
/ 02 августа 2020

Это то, что я сделал бы (не видя полного кода, трудно написать полное решение (также это съест много памяти, но опять же - чтобы оптимизировать это, мне нужно было бы увидеть код; то, что я пишу, имеет то преимущество, что его просто реализовать и не требуется блокировка).

Итак, рекомендация:

  1. создать cr2_vec = [copy(cr1) for i in 0:499]
  2. во внешнем l oop запись cr2 = cr2_vec[k]
  3. Затем выполните всю обработку на cr2
  4. после завершения @threads l oop возьмите cr2_vec, в котором будут обновлены кадры данных и из каждого фрейма данных этого вектора возьмите необходимые столбцы и добавьте их в исходный cr1 фрейм данных.

Более продвинутым решением было бы не использовать вектор фреймов данных, а только один кадр данных, как и вы, но после завершения вычислений используйте блокировку, а внутри блокировки обновите глобальный cr1 только с вычисленными столбцами (вам нужно использовать блокировку, чтобы избежать состояния гонки).

ИЗМЕНИТЬ

Пример достаточно для более эффективной реализации: 

~$ JULIA_NUM_THREADS=4 julia
               _
   _       _ _(_)_     |  Documentation: https://docs.julialang.org
  (_)     | (_) (_)    |
   _ _   _| |_  __ _   |  Type "?" for help, "]?" for Pkg help.
  | | | | | | |/ _` |  |
  | | |_| | | | (_| |  |  Version 1.4.2 (2020-05-23)
 _/ |\__'_|_|_|\__'_|  |  Official https://julialang.org/ release
|__/                   |

julia> using DataFrames

julia> function example()
           df = DataFrame(id=1:10)
           tmp_df = df[!, :] # avoid copying columns, just copy the structure
           l = Threads.ReentrantLock()
           Threads.@threads for i in 1:500
               # avoid copying columns, just copy the structure; I assume that you do not modidy existing columns
               # if you do write copy(tmp_df) instead
               local_df = tmp_df[!, :]
               local_df.x = rand(nrow(local_df)) # here use any operations you want that add columns to local_df
               Threads.lock(l)
               # and here add new columns to df in a thread safe way
               # I have made just a simple example
               df[!, string("col", i)] = local_df.x
               Threads.unlock(l)
           end
           return df
       end
example (generic function with 1 method)

julia> example()
10×501 DataFrame. Omitted printing of 488 columns
│ Row │ id    │ col126    │ col127    │ col128    │ col1     │ col376     │ col251    │ col129    │ col377   │ col252   │ col378    │ col130    │ col253   │
│     │ Int64 │ Float64   │ Float64   │ Float64   │ Float64  │ Float64    │ Float64   │ Float64   │ Float64  │ Float64  │ Float64   │ Float64   │ Float64  │
├─────┼───────┼───────────┼───────────┼───────────┼──────────┼────────────┼───────────┼───────────┼──────────┼──────────┼───────────┼───────────┼──────────┤
│ 1   │ 1     │ 0.0859079 │ 0.196322  │ 0.419307  │ 0.692845 │ 0.0649538  │ 0.271088  │ 0.061715  │ 0.580881 │ 0.962198 │ 0.799577  │ 0.168915  │ 0.147532 │
│ 2   │ 2     │ 0.943435  │ 0.367186  │ 0.999753  │ 0.955678 │ 0.0250972  │ 0.960388  │ 0.0186629 │ 0.360238 │ 0.482318 │ 0.630828  │ 0.638949  │ 0.078192 │
│ 3   │ 3     │ 0.47552   │ 0.73942   │ 0.81966   │ 0.354086 │ 0.00347341 │ 0.889408  │ 0.836998  │ 0.46696  │ 0.467453 │ 0.939678  │ 0.211346  │ 0.28351  │
│ 4   │ 4     │ 0.802997  │ 0.761556  │ 0.848161  │ 0.893623 │ 0.651274   │ 0.564887  │ 0.248991  │ 0.667879 │ 0.644271 │ 0.25262   │ 0.246616  │ 0.759315 │
│ 5   │ 5     │ 0.252624  │ 0.452797  │ 0.925494  │ 0.986887 │ 0.125357   │ 0.593984  │ 0.030369  │ 0.607307 │ 0.572907 │ 0.496147  │ 0.512023  │ 0.166856 │
│ 6   │ 6     │ 0.538396  │ 0.225402  │ 0.800646  │ 0.296837 │ 0.74292    │ 0.327231  │ 0.210256  │ 0.60462  │ 0.611555 │ 0.0593179 │ 0.662082  │ 0.932477 │
│ 7   │ 7     │ 0.0371803 │ 0.7965    │ 0.0444999 │ 0.178894 │ 0.106347   │ 0.780827  │ 0.637981  │ 0.237108 │ 0.627657 │ 0.944787  │ 0.378639  │ 0.878564 │
│ 8   │ 8     │ 0.811234  │ 0.187935  │ 0.227647  │ 0.262164 │ 0.620936   │ 0.0184777 │ 0.438722  │ 0.38764  │ 0.125682 │ 0.194858  │ 0.490097  │ 0.958929 │
│ 9   │ 9     │ 0.540951  │ 0.0577447 │ 0.0718953 │ 0.316012 │ 0.670796   │ 0.867076  │ 0.0432124 │ 0.929919 │ 0.272197 │ 0.410973  │ 0.455775  │ 0.21984  │
│ 10  │ 10    │ 0.471401  │ 0.185443  │ 0.817339  │ 0.757105 │ 0.717144   │ 0.257629  │ 0.528992  │ 0.347383 │ 0.912039 │ 0.905729  │ 0.0434874 │ 0.75463  │

Обратите внимание, что в результате порядок столбцов будет зависеть от того, в каком порядке выполняются параллельные потоки, поэтому, вероятно, вы захотите изменить порядок столбцов после этой операции вручную.

...