initzhang / ducati_sigmod Goto Github PK

$CUDA_VISIBLE_DEVICES=0 python run_allocate.py --dataset ogbn-papers100M --fanouts 10,25 --fake-dim 128
2023-09-23 14:24:59,560 Namespace(adj_budget=0, adj_slope=1, batches=1000, bs=8000, dataset='ogbn-papers100M', fake_dim=128, fanouts='10,25', nfeat_budget=0, nfeat_slope=1, pre_batches=100, pre_epochs=2, runs=4, total_budget=1)
2023-09-23 14:25:00,598 loading raw dataset of ogbn-papers100M
2023-09-23 14:25:38,354 finish loading raw dataset, time elapsed: 37.76s
2023-09-23 14:26:09,762 finish preprocessing, time elapsed: 31.41s
2023-09-23 14:28:15,314 finish generating random features with dim=128, time elapsed: 124.61s
2023-09-23 14:28:16,243 Graph(num_nodes=111059956, num_edges=1615685872,
      ndata_schemes={}
      edata_schemes={})
2023-09-23 14:28:16,454 get 1000 seeds, 0.06GB on cuda:0
2023-09-23 14:28:16,455 start profiling and calculating slope
2023-09-23 14:28:42,581 finish calculating slope: adj(2.55) nfeat(13.45), time elapsed: 26.13s
2023-09-23 14:28:42,581 total cache budget: 1GB
2023-09-23 14:28:42,581 total adj size: 12.865GB, total nfeat size: 53.785GB
2023-09-23 14:28:44,675 finish constructing density and size array
2023-09-23 14:28:57,285 find the separate point 4565543
2023-09-23 14:28:57,325 nfeat entries: 1770741, adj entries: 2794802
2023-09-23 14:28:57,325 nfeat size: 0.858 GB, adj size: 0.142 GB
2023-09-23 14:28:57,684 dual cache allocation done, time_elapsed: 15.10s
2023-09-23 14:28:58,128 current allocation plan: 0.142GB adj cache & 0.858GB nfeat cache

$CUDA_VISIBLE_DEVICES=0 python run_ducati.py

2023-09-23 14:42:06,607 Namespace(adj_budget=0, batches=1024, bs=8000, dataset='ogbn-papers100M', dropout=0.5, fake_dim=128, fanouts='10,25', lr=0.003, nfeat_budget=0, num_hidden=256, pre_batches=100, pre_epochs=2, runs=10)
2023-09-23 14:42:07,654 loading raw dataset of ogbn-papers100M
2023-09-23 14:42:45,337 finish loading raw dataset, time elapsed: 37.68s
2023-09-23 14:43:16,913 finish preprocessing, time elapsed: 31.58s
2023-09-23 14:45:22,367 finish generating random features with dim=128, time elapsed: 124.49s
2023-09-23 14:45:23,257 Graph(num_nodes=111059956, num_edges=1615685872,
      ndata_schemes={}
      edata_schemes={})
2023-09-23 14:45:23,485 get 1024 seeds, 0.06GB on cuda:0
gpu_flag None
gpu_map None
all_cache [None, None]
2023-09-23 14:45:23,882 buffer size: 0.185 GB
Traceback (most recent call last):
  File "run_ducati.py", line 109, in <module>
    entry(args, graph, all_data, seeds_list, counts)
  File "run_ducati.py", line 63, in entry
    run_one_list(seeds_list)
  File "run_ducati.py", line 49, in run_one_list
    cur_nfeat = nfeat_loader.load(input_nodes, nfeat_buf) # fetch nfeat
  File "/home/bear/workspace/DUCATI_SIGMOD/NfeatLoader.py", line 9, in load
    gpu_mask = self.gpu_flag[idx]
TypeError: 'NoneType' object is not subscriptable

def separate_features_idx(args, graph):
    separate_tic = time.time()
    train_idx = torch.nonzero(graph.ndata.pop("train_mask")).reshape(-1)
    adj_counts = graph.ndata.pop('adj_counts')
    nfeat_counts = graph.ndata.pop('nfeat_counts')

    # cleanup
    graph.ndata.clear()
    graph.edata.clear()

    # we prepare fake input for all datasets
    fake_nfeat = dgl.contrib.UnifiedTensor(torch.rand((graph.num_nodes(), args.fake_dim), dtype=torch.float), device='cuda')
    fake_label = dgl.contrib.UnifiedTensor(torch.randint(args.n_classes, (graph.num_nodes(), ), dtype=torch.long), device='cuda')

    mlog(f'finish generating random features with dim={args.fake_dim}, time elapsed: {time.time()-separate_tic:.2f}s')
    return graph, [fake_nfeat, fake_label], train_idx, [adj_counts, nfeat_counts]