你好，感谢你的开源。我在做do_evaluate和do_predict时，不管是你提供的数据，还是我自己的数据，都会出现错误，不知道这个问题时什么导致的，想请教一下你。

Traceback (most recent call last): File "Trainer.py", line 598, in main() File "Trainer.py", line 574, in main train(model, args, train_dataset, dev_dataset, test_dataset, label_vocab, tb_writer) File "Trainer.py", line 377, in train metrics, _ = evaluate(model, args, dev_dataset, label_vocab, global_step, description="Dev", write_file=True) File "Trainer.py", line 465, in evaluate all_label_ids, all_predict_ids, all_attention_mask, label_vocab) File "LEBERT/function/metrics.py", line 40, in seq_f1_with_mask tmp_pred.append(label_vocab.convert_id_to_item(all_pred_labels[i][j]).replace("M-", "I-")) File "LEBERT/feature/vocab.py", line 84, in convert_id_to_item return self.idx2item[id] IndexError: list index out of range

你好,大佬: 首先，感谢开源！ 尝试复现论文结果的时候遇到了一些问题,不知能否抽空解答一下. 1、在运行weiboNER的实验代码时，超参数设置与论文中一样，训练时loss下降有些异常(震荡下降，且前几个epoch验证集测试集f1均为0)，训练日志已邮件发送； 2、具体环境及运行设置： GPU：A100-SXM4-40GB； torch:1.8.1+cu111 训练方式：单卡 期待大佬回复指导，谢谢！

def evaluate(model, args, dataset, label_vocab, global_step, description="dev", write_file=False): """ evaluate the model's performance """ dataloader = get_dataloader(dataset, args, mode='dev') if (not args.do_train) and (not args.no_cuda) and args.local_rank != -1: model = model.cuda() model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True )

batch_size = dataloader.batch_size
if args.local_rank == 0 or args.local_rank == -1:
    logger.info("***** Running %s *****", description)
    logger.info("  Num examples = %d", len(dataloader.dataset))
    logger.info("  Batch size = %d", batch_size)
eval_losses = []
model.eval()

all_input_ids = None
all_label_ids = None
all_predict_ids = None
all_attention_mask = None

for batch in tqdm(dataloader, desc=description):
    # new batch data: [input_ids, token_type_ids, attention_mask, matched_word_ids,
    # matched_word_mask, boundary_ids, labels
    batch_data = (batch[0], batch[2], batch[1], batch[3], batch[4], batch[5], batch[6])
    new_batch = batch_data
    batch = tuple(t.to(args.device) for t in new_batch)
    inputs = {"input_ids": batch[0], "attention_mask": batch[1], "token_type_ids": batch[2],
              "matched_word_ids": batch[3], "matched_word_mask": batch[4],
              "boundary_ids": batch[5], "labels": batch[6], "flag": "Predict"}
    batch_data = None
    new_batch = None

    with torch.no_grad():
        outputs = model(**inputs)
        preds = outputs[0]

========================================================================= training has no problem，but weibo/labels.txt has 28 tags，two tags '' and ''，add to 30. but 31 in pred value.

O B-PER.NOM E-PER.NOM B-LOC.NAM E-LOC.NAM B-PER.NAM I-PER.NAM E-PER.NAM S-PER.NOM B-GPE.NAM E-GPE.NAM B-ORG.NAM I-ORG.NAM E-ORG.NAM I-PER.NOM S-GPE.NAM B-ORG.NOM E-ORG.NOM I-LOC.NAM I-ORG.NOM B-LOC.NOM I-LOC.NOM E-LOC.NOM B-GPE.NOM E-GPE.NOM I-GPE.NAM S-PER.NAM S-LOC.NOM

class ItemVocabFile(): """ Build vocab from file. Note, each line is a item in vocab, or each items[0] is in vocab """ def init(self, files, is_word=False, has_default=False, unk_num=0): self.files = files self.item2idx = {} self.idx2item = [] self.item_size = 0 self.is_word = is_word if not has_default and not self.is_word: self.item2idx[''] = self.item_size self.idx2item.append('') self.item_size += 1 self.item2idx[''] = self.item_size self.idx2item.append('') self.item_size += 1 # for unk words for i in range(unk_num): self.item2idx['{}'.format(i+1)] = self.item_size self.idx2item.append('{}'.format(i+1)) self.item_size += 1

    self.init_vocab()
    print('=======labels info========')
    print(self.item2idx)
    print(self.idx2item)

=======labels info======== {'': 0, '': 1, 'O': 2, 'B-PER.NOM': 3, 'E-PER.NOM': 4, 'B-LOC.NAM': 5, 'E-LOC.NAM': 6, 'B-PER.NAM': 7, 'I-PER.NAM': 8, 'E-PER.NAM': 9, 'S-PER.NOM': 10, 'B-GPE.NAM': 11, 'E-GPE.NAM': 12, 'B-ORG.NAM': 13, 'I-ORG.NAM': 14, 'E-ORG.NAM': 15, 'I-PER.NOM': 16, 'S-GPE.NAM': 17, 'B-ORG.NOM': 18, 'E-ORG.NOM': 19, 'I-LOC.NAM': 20, 'I-ORG.NOM': 21, 'B-LOC.NOM': 22, 'I-LOC.NOM': 23, 'E-LOC.NOM': 24, 'B-GPE.NOM': 25, 'E-GPE.NOM': 26, 'I-GPE.NAM': 27, 'S-PER.NAM': 28, 'S-LOC.NOM': 29} ['', '', 'O', 'B-PER.NOM', 'E-PER.NOM', 'B-LOC.NAM', 'E-LOC.NAM', 'B-PER.NAM', 'I-PER.NAM', 'E-PER.NAM', 'S-PER.NOM', 'B-GPE.NAM', 'E-GPE.NAM', 'B-ORG.NAM', 'I-ORG.NAM', 'E-ORG.NAM', 'I-PER.NOM', 'S-GPE.NAM', 'B-ORG.NOM', 'E-ORG.NOM', 'I-LOC.NAM', 'I-ORG.NOM', 'B-LOC.NOM', 'I-LOC.NOM', 'E-LOC.NOM', 'B-GPE.NOM', 'E-GPE.NOM', 'I-GPE.NAM', 'S-PER.NAM', 'S-LOC.NOM']

when I run run_ner.sh and --do_evaluate I have encountered these kind of problems：

Traceback (most recent call last):
  File "Trainer.py", line 598, in <module>
    main()
  File "Trainer.py", line 574, in main
    train(model, args, train_dataset, dev_dataset, test_dataset, label_vocab, tb_writer)
  File "Trainer.py", line 377, in train
    metrics, _ = evaluate(model, args, dev_dataset, label_vocab, global_step, description="Dev", write_file=True)
  File "Trainer.py", line 465, in evaluate
    all_label_ids, all_predict_ids, all_attention_mask, label_vocab)
  File "LEBERT/function/metrics.py", line 40, in seq_f1_with_mask
    tmp_pred.append(label_vocab.convert_id_to_item(all_pred_labels[i][j]).replace("M-", "I-"))
  File "LEBERT/feature/vocab.py", line 84, in convert_id_to_item
    return self.idx2item[id]
IndexError: list index out of range

/opt/conda/lib/python3.6/site-packages/transformers-4.7.0.dev0-py3.6.egg/transformers/tokenization_utils_base.py:1631: FutureWarning: Calling BertTokenizer.from_pretrained() with the path to a single file or url is deprecated and won't be possible anymore in v5. Use a model identifier or the path to a directory instead. FutureWarning, Traceback (most recent call last): File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main "main", mod_spec) File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code exec(code, run_globals) File "/opt/conda/lib/python3.6/site-packages/torch/distributed/launch.py", line 261, in main() File "/opt/conda/lib/python3.6/site-packages/torch/distributed/launch.py", line 257, in main cmd=cmd) subprocess.CalledProcessError: Command '['/opt/conda/bin/python', '-u', 'Trainer.py', '--local_rank=0', '--do_train', '--do_eval', '--do_predict', '--evaluate_during_training', '--data_dir=data/dataset/NER/weibo', '--output_dir=data/result/NER/weibo/lebertcrf', '--config_name=data/berts/bert/config.json', '--model_name_or_path=data/berts/bert/pytorch_model.bin', '--vocab_file=data/berts/bert/vocab.txt', '--word_vocab_file=data/vocab/tencent_vocab.txt', '--max_scan_num=100', '--max_word_num=5', '--label_file=data/dataset/NER/weibo/labels.txt', '--word_embedding=data/embedding/word_embedding.txt', '--saved_embedding_dir=data/dataset/NER/weibo', '--model_type=LEBertCRF_Token', '--seed=106524', '--per_gpu_train_batch_size=4', '--per_gpu_eval_batch_size=4', '--learning_rate=1e-5', '--max_steps=-1', '--max_seq_length=256', '--num_train_epochs=2', '--warmup_steps=190', '--save_steps=600', '--logging_steps=600']' died with <Signals.SIGKILL: 9>.

when I prepare all of dataset, and run ./run_ner.sh, it will throw the error:

Traceback (most recent call last):
  File "Trainer.py", line 593, in <module>
    main()
  File "Trainer.py", line 552, in main
    model = model.cuda()
UnboundLocalError: local variable 'model' referenced before assignment
Traceback (most recent call last):
  File "/home/human/miniconda3/envs/qzqExp/lib/python3.7/runpy.py", line 193, in _run_module_as_main

After checking the code, it's:

https://github.com/liuwei1206/LEBERT/blob/main/Trainer.py#L537-L550

there is no 'LEBertCRF_Token' in Trainner.

Hello, I have downloaded all the files you provide in the github, but I didn't find the checkpoints. Can you tell how to download it? Providing the links is the best, thank you.

Hi, I met some problems when replicating your experiments.

1. I use the ontonote4 checkpint you provided to do prediction. It shows that the crf.transitions has a shape torch.Size([20, 20]) from checkpoint, but actually this dataset only has 17 labels and the crf.transitions should be [19, 19]. So could you please check the checkpoint file of ontonote4?
2. I used a single GPU to train the models on weibo and ontonote4 dataset without changing any code or paramters. However the best F1 score of weibo is 0.68 and the ontonote4 is 0.80, which is lower than your result. If it is because you used the ditributed training, would you please provide the detailed paramters of distributed training, or the training parameters of single GPU that could reach your scores?

Many thanks in advance.

Your work is really amazing! I am currently learning your code. When I try to train LEBERT on the weibo dataset, I find that the predicted results are all "O" despite that I haven't done any changes to your code. However, using the checkpoint you provide can indeed get good results. What could be the reason for this? How can I train on my own to get these checkpoints provided by you? I would be very appreciated if you helped me!

@liuwei1206 您好，我的脚本参照给出的note4 shell设置如下: CUDA_VISIBLE_DEVICES=0 python3 -m torch.distributed.launch --master_port 13017 --nproc_per_node=1 \ Trainer.py --do_train --do_eval --do_predict --evaluate_during_training \ --data_dir="data/dataset/NER/note4" \ --output_dir="data/result/NER/note4/wcbertcrf" \ --config_name="data/berts/bert/config.json" \ --model_name_or_path="/home/root1/lizheng/pretrainModels/torch/chinese/bert-base-chinese/pytorch_model.bin" \ --vocab_file="/home/root1/lizheng/pretrainModels/torch/chinese/bert-base-chinese/vocab.txt" \ --word_vocab_file="data/vocab/tencent_vocab.txt" \ --max_scan_num=1500000 \ --max_word_num=5 \ --label_file="data/dataset/NER/note4/labels.txt" \ --word_embedding="data/embedding/word_embedding.txt" \ --saved_embedding_dir="data/dataset/NER/note4" \ --model_type="WCBertCRF_Token" \ --seed=106524 \ --per_gpu_train_batch_size=4 \ --per_gpu_eval_batch_size=32 \ --learning_rate=1e-5 \ --max_steps=-1 \ --max_seq_length=256 \ --num_train_epochs=20 \ --warmup_steps=190 \ --save_steps=600 \ --logging_steps=300

但结果test F1为80左右，是否因为您是多卡训练，我是单卡训练的差异，可否看一下脚本是否无误

Here's the running error.

Calling BertTokenizer.from_pretrained() with the path to a single file or url is deprecated Traceback (most recent call last): File "/home/wen/anaconda3/envs/pytorch/lib/python3.7/runpy.py", line 193, in _run_module_as_main "main", mod_spec) File "/home/wen/anaconda3/envs/pytorch/lib/python3.7/runpy.py", line 85, in _run_code exec(code, run_globals) File "/home/wen/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/distributed/launch.py", line 261, in main() File "/home/wen/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/distributed/launch.py", line 257, in main cmd=cmd) subprocess.CalledProcessError: Command '['/home/wen/anaconda3/envs/pytorch/bin/python3', '-u', 'Trainer.py', '--local_rank=0', '--do_train', '--do_eval', '--do_predict', '--evaluate_during_training', '--data_dir=data/dataset/COIE/origin', '--output_dir=data/result/COIE/origin/lebertcrf', '--config_name=data/berts/bert/config.json', '--model_name_or_path=data/berts/bert/pytorch_model.bin', '--vocab_file=data/berts/bert/vocab.txt', '--word_vocab_file=data/vocab/tencent_vocab.txt', '--max_scan_num=1000000', '--max_word_num=5', '--label_file=data/dataset/COIE/origin/labels.txt', '--word_embedding=data/embedding/Tencent_AILab_ChineseEmbedding.txt', '--saved_embedding_dir=data/dataset/COIE/origin', '--model_type=WCBertCRF_Token', '--seed=106524', '--per_gpu_train_batch_size=4', '--per_gpu_eval_batch_size=16', '--learning_rate=1e-5', '--max_steps=-1', '--max_seq_length=256', '--num_train_epochs=20', '--warmup_steps=190', '--save_steps=600', '--logging_steps=100']' died with <Signals.SIGKILL: 9>.

And here's the log.

2021-07-07 15:40:29:INFO: Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, 16-bits training: False 2021-07-07 15:40:29:INFO: Training/evaluation parameters Namespace(adam_epsilon=1e-08, config_name='data/berts/bert/config.json', data_dir='data/dataset/COIE/origin', default_label='O', device=device(type='cuda', index=0), do_eval=True, do_predict=True, do_shuffle=True, do_train=True, evaluate_during_training=True, fp16=False, fp16_opt_level='O1', gradient_accumulation_steps=1, label_file='data/dataset/COIE/origin/labels.txt', learning_rate=1e-05, local_rank=0, logging_dir='data/log', logging_steps=100, max_grad_norm=1.0, max_scan_num=1000000, max_seq_length=256, max_steps=-1, max_word_num=5, model_name_or_path='data/berts/bert/pytorch_model.bin', model_type='WCBertCRF_Token', n_gpu=1, no_cuda=False, nodes=1, num_train_epochs=20, output_dir='data/result/COIE/origin/lebertcrf', overwrite_cache=True, per_gpu_eval_batch_size=16, per_gpu_train_batch_size=4, save_steps=600, save_total_limit=50, saved_embedding_dir='data/dataset/COIE/origin', seed=106524, sgd_momentum=0.9, vocab_file='data/berts/bert/vocab.txt', warmup_steps=190, weight_decay=0.0, word_embed_dim=200, word_embedding='data/embedding/Tencent_AILab_ChineseEmbedding.txt', word_vocab_file='data/vocab/tencent_vocab.txt')

Hope you can reply. Thx.