TBgen_App/autoline/__pycache__/TB4_eval.cpython-312.pyc

<EFBFBD>
P<>Yi6/<00><00>r<00>dZd
dl
Z
d
dlZd
dlZd
dlmZ
d
dlm	Z	
d
dl
mZmZ
gd<06>
Z
ddgZd	g
ZGd
<EFBFBD>d<0B>Zy)z<>
Description :   This is the testbench eval stage in autoline
Author      :   Ruidi Qiu (r.qiu@tum.de)
Time        :   2024/7/24 11:24:43
LastEdited  :   2024/8/28 21:08:21
<EFBFBD>N)
<01>
autologger)
<01>log_localprefix)<02>Timer<65>get_time)zAll test cases passedzall test cases passedzAll Test Cases PassedzMismatches: 0 in z*Hint: Total mismatched samples is 0 out ofz[]c<00><><00>eZ
dZd
Z	ddedededededed	ed
ededefd
<0A>Zed<0E>
d<0F><00>Z	d<10>Z
ddefd<12>Zdd<13>
Zd<14>Z
ed<15><00>Zed<16><00>Zed<17><00>Zeddedefd<1A><05>Zy)<1F>
TaskTBevalau
    ### description
    - this is the evaluation stage of our pipeline; the priority of this stage is that TB is generated and the empty DUT compilation is passed;
    - please use `try` to catch the exception of this function.
    - this module is independent from the previous modules.
    #### input
    - task_id: the name of the problem
    - root_dir: the dir of one problem
    - TB_gen: the testbench under evaluation (str)
    - TB_golden: the golden testbench (str)
    - DUT_golden: the golden RTL DUT (str)
    - DUT_mutant_list: the list of RTL DUT mutants modified from DUT_golden;[str]        
    #### output
    - dict
        - "Eval1_pass" : bool (whether the golden RTL checking passed)
        - "Eval2_pass" : bool (whether the golden TB comparison on RTL mutants passed)
        - "Eval2_failed_mutant_idxes" : list of int (the index of the failed mutants)
    N<>task_id<69>task_dir<69>TB_gen<65>	TB_golden<65>
DUT_golden<EFBFBD>DUT_mutant_list<73>DUT_gptgen_list<73>pychecker_en<65>pychecker_code<64>
runfiles_savec<00>@<00>|
|_||_
||_||_||_||_||_||_|
|_|jsd
n
d|_	|	|_
d|_d|_tjj|d<05>|_d|_d|_d|_tjj|d<06>|_d|_d|_d|_d|_tjj|d<07>|_d|_d|_d|_y)Nr<00>	Pychecker<65>F<>eval1_GoldenRTL<54>eval2_GoldenTB_and_mutants<74>eval2b_GPTgenTB)r	r
rrr
rrr<00>save_en<65>TB_gen_moder<00>working_dir<69>Eval1_exist<73>os<6F>path<74>join<69>	Eval1_dir<69>
Eval1_results<74>
Eval1_pass<EFBFBD>Eval2_exist<73>	Eval2_dir<69>
Eval2_pass<EFBFBD>Eval2_failed_mutant_idx<64>Eval2_passed_mutant_idx<64>Eval2b_exist<73>
Eval2b_dir<EFBFBD>Eval2b_pass<73>Eval2b_failed_mutant_idx<64>Eval2b_passed_mutant_idx)<0B>selfr	r
rrr
rrrrrs           <20>7/home/zhang/CorrectBench/TBgen_App/autoline/TB4_eval.py<70>__init__zTaskTBeval.__init__(s

<00><00><1E><04><0C> <20><04>
<0A><1C><04><0B>"<22><04><0E>$<24><04><0F>.<2E><04><1C>.<2E><04><1C>(<28><04><19>$<24><04><0C>+/<2F>+<<3C>+<<3C>8<EFBFBD>+<2B><04><18>,<2C><04><1B><1D><04><18> <20><04><18><1B><17><17><1C><1C>h<EFBFBD>0A<30>B<><04><0E>!<21><04><1A><1E><04><0F> <20><04><18><1B><17><17><1C><1C>h<EFBFBD>0L<30>M<><04><0E><1E><04><0F>'+<2B><04>$<24>'+<2B><04>$<24>!<21><04><19><1C>'<27>'<27>,<2C>,<2C>x<EFBFBD>1B<31>C<><04><0F><1F><04><18>(,<2C><04>%<25>(,<2C><04>%<25><00>TBevalc
<00>b
<00>|j<00>|j<00>
|jrU|j<00>|j<00>|jd
<01><02>

|j<00>A|j<00>5|jd<03><02>

n"tjd|jz<00>

|j<00>
y)N<>mutant)
<01>mode<64>gptgenz/[%s] Eval 2/2b is skipped because Eval 1 failed)r
<00>	run_Eval1r"rr<00>	run_Eval2r<00>logger<65>infor	<00>clean_wave_vcd<63>
r-s
 r.<00>runzTaskTBeval.runJs<><00><00><10>?<3F>?<3F>&<26><10>N<EFBFBD>N<EFBFBD><1C><0F>?<3F>?<3F><13>~<7E>~<7E>)<29>d<EFBFBD>.B<>.B<>.N<><14><0E><0E>H<EFBFBD><0E>-<2D><13>~<7E>~<7E>)<29>d<EFBFBD>.B<>.B<>.N<><14><0E><0E>H<EFBFBD><0E>-<2D><12>K<EFBFBD>K<EFBFBD>I<>T<EFBFBD>\<5C>\<5C>Z<>[<5B><0C><1B><1B>r0c
	<00>X
<00>d
}
t
jd<02>

|j|j|j|j
|j|jd
|j<00><03>|_	t
j|jddd|jrdn
dz<00>
d
|_y)NTz"Eval 1: Golden RTL checking begins)<02>raise_when_failr<00>positive<76>failedzEval 1: Golden RTL checking %s!<21>passed)r8r9<00>
run_testbenchr rr
rrrr"<00>match_levelr)r-<00>silents  r.r6zTaskTBeval.run_Eval1Zs<><00><00><15><06><0E><0B><0B>8<>9<><1E>,<2C>,<2C>T<EFBFBD>^<5E>^<5E>T<EFBFBD>[<5B>[<5B>$<24>/<2F>/<2F>[_<>[k<>[k<>mq<6D>n
A<02>n
A<02>SW<02>ae<02>am<02>am<02>,<2C>n<02><04><0F><0E><1A><1A>4<EFBFBD>?<3F>?<3F>J<EFBFBD><08>Bc<42>sw<73>t
C<02>t
C<02>go<67>IQ<02>C
R<02>	S<02> <20><04>r0r4c	<00>(<00>d
}|
dvs
Jd|
z<00><00>
|
dk(rd}d}|j}|j}n!|
dk(rd}d}|j}|j}t	j
<03>

g}t
<05>
D]<5D>\}}	tjjd	|d
zfz<00>}
tjj|
d<0B>}tjj|
d<0C>}	|j||j|	d
|j<00><0E>}
	|j||j|	|j|j|j<00><0E>}|
s|sd
}n	|
r|rd
}nd}|j!|<0F>

<00><>t#|<07>
}t
|<07>
D<00><08>cgc]
\}}|r
<01>	|d
z<00><02>}}}t
|<07>
D<00><08>cgc]
\}}|s
<01>	|d
z<00><02>}}}|
dk(r |||d
f\|_|_|_|_n$|
dk(r|||d
f\|_|_|_|_|rdndt5|<13>
t5|<07>
fz}|st5|<13>
t5|<07>
zdk\rtj6ntj8}||<03>
d|<11>
d<14><04>

y#
d}
Y<00>
<01>lxYw
#
d}Y<00>
<01>6xYw
c
c}}wc
c}}w)z mode: "mutant" or "gptgen" T)r3r5zInvalid mode in run_Eval2: r3z)Eval 2: Golden TB checking on RTL mutantsr5z6Eval 2b: Golden TB checking on GPT generated RTL codes<65>
gptgen_DUTz%s_%d<>
<00>GoldenTB<54>GeneratedTBr)
rFzperfectly passedzfinished (%d/%d)g<><67><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>?<3F>
 <20>
!N)rr$rr)r8r9<00>	enumeraterrrrBrrrrr<00>append<6E>allr%r&r'r#r*r+r,r(<00>len<65>successr@)r-r4rD<00>	print_str<74>mutant_subdir_name<6D>DUT_list<73>eval_dir<69>mutant_results<74>idx<64>
DUT_mutant<EFBFBD>
mutant_subdir<69>GoldenTB_subsubdir<69>GenedTB_subsubdir<69>
TBgolden_pass<73>
TBgen_pass<EFBFBD>mutant_pass<73>	eval_pass<73>result<6C>failed_mutant_idx<64>passed_mutant_idx<64>my_logs                     r.r7zTaskTBeval.run_Eval2ds<><00><00><15><06><13>+<2B>+<2B>Q<>-J<>T<EFBFBD>-Q<>Q<>+<2B><0F>8<EFBFBD><1B>C<>I<EFBFBD>!)<29><1E><1B>+<2B>+<2B>H<EFBFBD><1B>~<7E>~<7E>H<EFBFBD>
<11>X<EFBFBD>
<1D>P<>I<EFBFBD>!-<2D><1E><1B>+<2B>+<2B>H<EFBFBD><1B><EFBFBD><EFBFBD>H<EFBFBD><0E><0B><0B>I<EFBFBD><1E><1B><0E>(<28><18>2<>	/<2F>O<EFBFBD>C<EFBFBD><1A><1E>G<EFBFBD>G<EFBFBD>L<EFBFBD>L<EFBFBD><18>7<EFBFBD><N<>PS<50>TU<54>PU<50>;V<>3V<33>W<>M<EFBFBD>!#<23><17><17><1C><1C>m<EFBFBD>Z<EFBFBD>!H<><1E> "<22><07><07><0C><0C>]<5D>M<EFBFBD> J<><1D>
&<26> $<24> 2<> 2<>3E<33>t<EFBFBD>~<7E>~<7E>Wa<57>cn<63>x|<7C>y
E<02>y
E<02> 2<>!F<02>
<0A>
#<23>!<21>/<2F>/<2F>0A<30>4<EFBFBD>;<3B>;<3B>PZ<50>\`<60>\l<>\l<>nr<6E>o
B<02>o
B<02>LP<02>LX<02>LX<02>/<2F>Y<02>
<EFBFBD>!<21><1A>"<22><0B><1E>:<3A>"<22><0B>#<23><0B><1A>!<21>!<21>+<2B>.<2E>+	/<2F>,<18><0E>'<27>	<09>8A<38>.<2E>8Q<38>`<60><1B><13>f<EFBFBD>Y_<59>S<EFBFBD>1<EFBFBD>W<EFBFBD>`<60><19>`<60>8A<38>.<2E>8Q<38>\<5C><1B><13>f<EFBFBD>U[<5B>S<EFBFBD>1<EFBFBD>W<EFBFBD>\<5C><19>\<5C><0F>8<EFBFBD><1B>lu<6C>x
I<02>K\<02>^b<02>m
b<02>i<>D<EFBFBD>O<EFBFBD>T<EFBFBD>9<>4<EFBFBD>;W<>Y]<5D>Yi<59>
<11>X<EFBFBD>
<1D>py<70>|
M<02>O`<02>bf<02>q
f<02>m<>D<EFBFBD><1C>d<EFBFBD>;<3B>T<EFBFBD>=Z<>\`<60>\m<>'0<>#<23>7I<37>S<EFBFBD>Qb<51>Mc<4D>eh<65>iw<69>ex<65>Ly<4C>7y<37><06>$-<2D>#<23>6G<36>2H<32><13>^<5E>I\<5C>2\<5C>^a<>2a<32><16><1E><1E>io<69>iv<69>iv<69><06><0E>9<EFBFBD>f<EFBFBD>-<2D>.<2E><>-

&<26> %<25>
<0A><>

#<23>"<22>
<EFBFBD><EFBFBD>a
<01><>\s0<00>%*I4<02>?I><02>
J<06>J<06>.
J<06>9J<06>4I;<05>>Jc<00>V<00>|d
vs
Jd|z<00><00>
t
j|
d<03><04>
|
|_t|jd<05>5}|j|<02>

ddd<06>
t|jd<05>5}|j|<03>

ddd<06>
tj|
d<03><07>}	|r|	ds
Jd	|z<00><00>
|d
k(r<>t|jd<05>5}|j|<05>

ddd<06>
tj|jd<03><0B>}
|r|
ds
Jd|z<00><00>
|jd|
d
dd
<EFBFBD><0F>|	dz
|
dz
}n |jd|	dd|<04><0F>|	dz
}|sd|
<01>d<12>dz}t
j|<0C>

|S#1s
w
Y

<00>
<01>2xYw
#1s
w
Y

<00>
<01>xYw
#1s
w
Y

<00><>xYw
)a<>

        it has two mode: pychecker mode or verilog testbench mode
        -input:
            - dir: the dir to save the TB, DUT and pychecker code
            - TB_code: str; the testbench code
            - DUT_code: str; the DUT code
            - TB_type: str: TB_gen, TB_golden, Pychecker
            - pychecker_code: str; the pychecker code
        - output:
            - pass: bool; if the DUT passed the testbench
        <20>rrrz"Invalid TB_type in run_testbench: T)
<01>exist_ok<6F>
wN)
rDrzh%s Iverilog Compilation Failed: the PREREQUISITE of 'Evaluation' is no syntactic error from Testbench!!!r)<02>pypathrDzh%s Python Compilation Failed: the PREREQUISITE of 'Evaluation' is no syntactic error from Python code!!!rG<00>out)<03>sim_pass<73>sim_out<75>TB_type<70>zfind z -type f ! -name 'run_info*'z -exec rm -f {} +)r<00>makedirsr<00>open<65>TB_path<74>write<74>DUT_path<74>iv<69>iverilog_call_and_save<76>PY_path<74>py<70>python_call_and_save<76>TC_pass_from_TC_out<75>system)
r-<00>dir<69>TB_code<64>DUT_coderkrr>r<00>
f<>iv_run_info<66>py_run_info<66>TC_pass<73>cmds
             r.rBzTaskTBeval.run_testbench<63>s<>
<00><00><17>><3E>><3E>n<>@d<>gn<67>@n<>n<>><3E>
<EFBFBD><0B><0B>C<EFBFBD>$<24>'<27><1E><04><18>
<11>$<24>,<2C>,<2C><03>
$<24>
	<1D>
<01>
<0A>G<EFBFBD>G<EFBFBD>G<EFBFBD><1C>
	<1D>
<11>$<24>-<2D>-<2D><13>
%<25>
	<1E><11>
<0A>G<EFBFBD>G<EFBFBD>H<EFBFBD><1D>
	<1E><18>/<2F>/<2F><03>D<EFBFBD>A<><0B><1A><1E>q<EFBFBD>><3E>
X<02>$N<02>PW<02>$X<02>
X<02>><3E><12>k<EFBFBD>!<21><15>d<EFBFBD>l<EFBFBD>l<EFBFBD>C<EFBFBD>(<28>

(<28>A<EFBFBD><11><07><07><0E>'<27>

(<28><1C>1<>1<><14><1C><1C>d<EFBFBD>S<>K<EFBFBD><1E>"<22>1<EFBFBD>~<7E>\<02>(R<02>T[<02>(\<02>\<02>~<7E><1A>.<2E>.<2E><04>k<EFBFBD>RS<52>n<EFBFBD>UZ<55>F[<5B>ep<65>.<2E>q<>t<74>AB<02>u
C<02>C<02>FQ<02>RS<02>FT<02>T<02>G<EFBFBD><1A>.<2E>.<2E><04>k<EFBFBD>RS<52>n<EFBFBD>UZ<55>F[<5B>el<65>.<2E>m<>p{<7B>|}<7D>p~<7E>~<7E>G<EFBFBD><16><19>#<23><15>:<3A>;<3B>>R<>R<>C<EFBFBD><0E>I<EFBFBD>I<EFBFBD>c<EFBFBD>N<EFBFBD><16><0E>-
	<1D>
	<1D><>
	<1E>
	<1E><>

(<28>

(<28>s$<00>F<03>3F<03>F<03>F<07>F<07>F(c
<00><><00>|j}
tj|
<01>
D]R\}}}|D]G}|jd
<01>
s
<01>tjtj
j
||<05><00>

<00>I<00>Ty)z$clean the .vcd files in the task_dirz.vcdN)r
r<00>walk<6C>endswith<74>removerr)r-<00>	clean_dir<69>root<6F>dirs<72>files<65>files      r.r:zTaskTBeval.clean_wave_vcd<63>sb<00><00><19>M<EFBFBD>M<EFBFBD>	<09>!#<23><17><17><19>!3<>	8<><1D>D<EFBFBD>$<24><05><1D>
8<><04><17>=<3D>=<3D><16>(<28><16>I<EFBFBD>I<EFBFBD>b<EFBFBD>g<EFBFBD>g<EFBFBD>l<EFBFBD>l<EFBFBD>4<EFBFBD><14>6<>7<>
8<>	8r0c
<00>p<00>tjj|j|jd
z<00>S)Nz_tb.v<>rrrrr	r;s
 r.rozTaskTBeval.TB_path<74>s(<00><00><12>w<EFBFBD>w<EFBFBD>|<7C>|<7C>D<EFBFBD>,<2C>,<2C>d<EFBFBD>l<EFBFBD>l<EFBFBD>W<EFBFBD>.D<>E<>Er0c
<00>p<00>tjj|j|jd
z<00>S)Nz.vr<76>r;s
 r.rqzTaskTBeval.DUT_path<74>s(<00><00><12>w<EFBFBD>w<EFBFBD>|<7C>|<7C>D<EFBFBD>,<2C>,<2C>d<EFBFBD>l<EFBFBD>l<EFBFBD>T<EFBFBD>.A<>B<>Br0c
<00>p<00>tjj|j|jd
z<00>S)Nz_tb.pyr<79>r;s
 r.rtzTaskTBeval.PY_path<74>s(<00><00><12>w<EFBFBD>w<EFBFBD>|<7C>|<7C>D<EFBFBD>,<2C>,<2C>d<EFBFBD>l<EFBFBD>l<EFBFBD>X<EFBFBD>.E<>F<>Fr0rirjc<00><><00>|s
y
|dvs
Jd|z<00><00>
tttd<02>}||}|dvr|D]}||
vs
<01>
yy
|dvr,|
jd<07>
}|
jd<08>
}||z
d	k(r
yy
y
)aQ

        get the information if DUT passed all the test cases from the testbench
        #### input
        - sim_pass: bool; if TB passed the compilation. if not, will return False without check
        - sim_out: the simulation output message;
        - TB_ty: "TB_gen" or "TB_golden" or "Pychecker"; the type of the testbench
        Frdz.Invalid TB_type during 'TC_pass_from_TC_out': )rrT)
r<00>
]<5D>
[rGN)<04>TC_PASS_CHECK_LIST_TB_GEN<45>TC_PASS_CHECK_LIST_TB_GOLDEN<45>TC_PASS_CHECK_LIST_PYCHECKER<45>rfind)rirjrk<00>tc_pass_check_list_dict<63>tc_pass_check_list<73>	check_str<74>last_bracket_end<6E>last_bracket_starts        r.rwzTaskTBeval.TC_pass_from_TC_out<75>s<><00><00><18><18><16>><3E>><3E>z<>@p<>sz<73>@z<>z<>><3E>-F<>Uq<55>A]<02>#^<02><1F>4<>W<EFBFBD>=<3D><1A><12>-<2D>-<2D>/<2F>
 <20>	<09><1C><07>'<27><1F>
 <20><19>
<14>
<0A>
%<25> '<27>}<7D>}<7D>S<EFBFBD>1<><1C>!(<28><1D><1D>s<EFBFBD>!3<><1E> <20>#5<>5<>!<21>;<3B><1B><1C>&r0)NNNNFrT)
r3)rFT)
r)<14>__name__<5F>
__module__<EFBFBD>__qualname__<5F>__doc__<5F>str<74>list<73>boolr/rr<r6r7rBr:<00>propertyrorqrt<00>staticmethodrw<00>r0r.rrs
<00><00><08>$:<3A> -<2D><03> -<2D>s<EFBFBD> -<2D>C<EFBFBD> -<2D>3<EFBFBD> -<2D>`c<> -<2D>z~<7E> -<2D>VZ<02> -<2D>pt<02> -<2D>MP<03> -<2D>ei<03> -<2D>D
<15>X<EFBFBD><1E>
<1E><1F>
<1E> <20>0/<2F>S<EFBFBD>0/<2F>d
'<17>R
8<><0E>F
<01><0E>F
<01><0E>C
<01><0E>C
<01><0E>G
<01><0E>G
<01><12><1D>d<EFBFBD><1D>S<EFBFBD><1D><12>r0r)r<>r<00>
iverilog_callrr<00>python_callru<00>loader_saverrr8r<00>utils.utilsrrr<>r<>r<>rr<>r0r.<00><module>r<>
sD<00>


<01>
<04>

<EFBFBD><1A><18>-<2D>(<28>'<27>g<><19> 3<>5a<35>b<><1C> $<24>v<EFBFBD><1C>a
<1D>a
r0