The performance of the trajectory is inconsistent with the paper about tree-of-thought-llm

[ysymyth]

Game24 in Trajectory
Standard: 33 (×)
CoT: 49 (×)

that's probably pass@100? see figure 3 in paper

Crosswords in Trajectory
Tot: 69 44 0 (×)

which trajectory file did you use? that's more like the "-prune" result in table 3

[li-aolong]

Game24 in Trajectory
Standard: 33 (×)
CoT: 49 (×)

yes, it's pass@100, but ToT is 0.238.

Crosswords in Trajectory
Tot: 69 44 0 (×)

I used infoss_dfs_prune.json

Here is my evaluate code and results

import json

def parse_game24(file_path):
    datas = json.load(open(file_path))
    if 'accs' in datas[0].keys():
        accs = [data['accs'] for data in datas]
    if 'infos' in datas[0].keys():
        accs = [[acc['r'] for acc in data['infos']] for data in datas]
    results = [(sum(acc)/len(acc), any(acc)) for acc in accs]
    avg_trial_acc = sum([result[0] for result in results]) / len(results)
    avg_task_acc = sum([result[1] for result in results]) / len(results)
    print(f'{avg_trial_acc:.3f}', avg_task_acc)
    
def parse_crosswords(file_path):
    datas = json.load(open(file_path))
    if 'dfs' not in file_path:
        accs = [[(acc['r_letter'], acc['r_word']) for acc in data['infos']] for data in datas]
        letter_accs = [a[0] for acc in accs for a in acc]
        word_accs = [a[1] for acc in accs for a in acc]
        game_accs = [1 if acc == 1.0 else 0 for acc in word_accs]

        print(f'{sum(letter_accs)/len(letter_accs):.3f}', f'{sum(word_accs)/len(word_accs):.3f}', f'{sum(game_accs)/len(game_accs):.3f}')
    else:
        rates = [data[-1]['info'] for data in datas]
        # print(rates)
        avg_letter_acc = sum([rate['r_letter'] for rate in rates]) / len(rates)
        avg_word_acc = sum([rate['r_word'] for rate in rates]) / len(rates)
        avg_game_acc = sum([1 if rate['r_word'] == 1.0 else 0 for rate in rates]) / len(rates)
        print(f'{avg_letter_acc:.3f}', f'{avg_word_acc:.3f}', avg_game_acc)

print('paper game24:')
parse_game24('logs/game24/gpt-4_0.7_naive_standard_sample_100_start900_end1000.json')
parse_game24('logs/game24/gpt-4_0.7_naive_cot_sample_100_start900_end1000.json')
parse_game24('logs/game24/gpt-4_0.7_propose1_value3_greedy5_start900_end1000.json')

print('paper crosswords:')
parse_crosswords('logs/crosswords/gpt-4_0.7_naive_standard_sample_10_start0_end20.json')
parse_crosswords('logs/crosswords/gpt-4_0.7_naive_cot_sample_10_start0_end20.json')
parse_crosswords('logs/crosswords/infoss_dfs_prune.json')
parse_crosswords('logs/crosswords/infoss_dfs_no_prune.json')

results:

paper game24:
0.073 0.33
0.040 0.49
0.238 0.69

paper crosswords:
0.387 0.140 0.000
0.406 0.157 0.010
0.690 0.440 0.0
0.588 0.320 0.05

[ysymyth]

@li-aolong , thanks for providing the code!

1. in game24, at the end of tot-bfs we do not just uniformly pick one of the five beams (they are by design sorted). Instead, we can use a simple heuristic (e.g. output contains "Answer" and "Answer" uses all input numbers) to pick the best beam.
2. in crosswords, at the end of tot-dos, we do not pick the LAST visited node, but the node with max depth (break tie by choose the first visited node).

so your code should be changed to

import json
import re

def get_tot_game24_result(data):  # retrun the first output that has "answer" and uses all input numbers
    xs = data['steps'][0]['x'].split()
    for i in range(5):
        y = data['ys'][i]
        if 'Answer: ' in y:
            eq = y.split('Answer: ')[-1].split(' = ')[0]
            if sorted(re.findall(r'\d+', eq)) == sorted(xs):
                return data['infos'][i]['r']
    return 0

def parse_game24(file_path, method):
    datas = json.load(open(file_path))
    if 'accs' in datas[0].keys():
        accs = [data['accs'] for data in datas]
    if 'infos' in datas[0].keys():
        accs = [[acc['r'] for acc in data['infos']] for data in datas]
    if method == 'tot-bfs':
        results = [get_tot_game24_result(data) for data in datas]
        avg_trial_acc = sum(results) / len(results)
        print(f'{method} {avg_trial_acc:.3f}', 'NA')
    else:
        results = [(sum(acc)/len(acc), any(acc)) for acc in accs]
        avg_trial_acc = sum([result[0] for result in results]) / len(results)
        avg_task_acc = sum([result[1] for result in results]) / len(results)
        print(f'{method} {avg_trial_acc:.3f}', avg_task_acc)
    

def get_tot_crosswords_result(data):  # choose the first node with max depth
    max_env_step, idx = 0, 0
    for i in range(len(data)):
        if data[i]['env_step'] > max_env_step:
            max_env_step = data[i]['env_step']
            idx = i
    return data[idx]['info']

def parse_crosswords(file_path, method):
    datas = json.load(open(file_path))
    if 'dfs' not in file_path:
        accs = [[(acc['r_letter'], acc['r_word']) for acc in data['infos']] for data in datas]
        letter_accs = [a[0] for acc in accs for a in acc]
        word_accs = [a[1] for acc in accs for a in acc]
        game_accs = [1 if acc == 1.0 else 0 for acc in word_accs]

        print(f'{method} {sum(letter_accs)/len(letter_accs):.3f}', f'{sum(word_accs)/len(word_accs):.3f}', f'{sum(game_accs)/len(game_accs):.3f}')
    else:
        # rates = [data[-1]['info'] for data in datas]
        rates = [get_tot_crosswords_result(data) for data in datas]
        # print(rates)
        avg_letter_acc = sum([rate['r_letter'] for rate in rates]) / len(rates)
        avg_word_acc = sum([rate['r_word'] for rate in rates]) / len(rates)
        avg_game_acc = sum([1 if rate['r_word'] == 1.0 else 0 for rate in rates]) / len(rates)
        print(f'{method} {avg_letter_acc:.3f}', f'{avg_word_acc:.3f}', avg_game_acc)

print('game24 (pass@1, pass@100):')
parse_game24('logs/game24/gpt-4_0.7_naive_standard_sample_100_start900_end1000.json', 'io')
parse_game24('logs/game24/gpt-4_0.7_naive_cot_sample_100_start900_end1000.json', 'cot')
parse_game24('logs/game24/gpt-4_0.7_propose1_value3_greedy5_start900_end1000.json', 'tot-bfs')

print('paper crosswords (r_letter, r_word, r_game):')
parse_crosswords('logs/crosswords/gpt-4_0.7_naive_standard_sample_10_start0_end20.json', 'io')
parse_crosswords('logs/crosswords/gpt-4_0.7_naive_cot_sample_10_start0_end20.json', 'cot')
parse_crosswords('logs/crosswords/infoss_dfs_prune.json', 'tot-dfs (prune)')
parse_crosswords('logs/crosswords/infoss_dfs_no_prune.json', 'tot-dfs (no prune)')

and results:

game24 (pass@1, pass@100):
io 0.073 0.33
cot 0.040 0.49
tot-bfs 0.690 NA
paper crosswords (r_letter, r_word, r_game):
io 0.387 0.140 0.000
cot 0.406 0.157 0.010
tot-dfs (prune) 0.780 0.600 0.2
tot-dfs (no prune) 0.654 0.415 0.05

I will update the codebase to include such code to get paper table results. Thanks very much!