pip install transformers
pip install datasets

pip install minio

from datasets import load_dataset
emotions = load_dataset('emotions')

type(emotions)

datasets.dataset_dict.DatasetDict

emotions['train'].column_names

['text', 'label']

['sadness', 'joy', 'love', 'anger', 'fear', 'surprise']

bucket_name = 'emotions-data'
for split, dataset in emotions.items():
  dataset.to_json(f'emotions-{split}.jsonl')
  put_file(bucket_name, f'emotions-{split}.jsonl', f'emotions-{split}.jsonl')

data_files = {}
for split in ['train', 'validation', 'test']:
  data_files[split] = f'emotions-{split}.jsonl'
  get_object(bucket_name, f'emotions-{split}.jsonl', f'emotions-{split}.jsonl')

emotions = load_dataset('json', data_files=data_files)

def put_text(bucket_name: str, object_name: str, text: str) -> ObjectWriteResult:
  '''
  此函数将文件上传到 MinIO 并返回 object_info。
  '''

  # 加载凭据和连接信息。
  with open('credentials.json') as f:
    credentials = json.load(f)

  # 使用访问密钥和密钥创建客户端
  client = Minio(credentials['url'], # host.docker.internal
                credentials['accessKey'],
                credentials['secretKey'],
                secure=False)
  # 确保存储桶存在。
  found = client.bucket_exists(bucket_name)
  if not found:
    client.make_bucket(bucket_name)

  # 上传文本。
  text_as_bytes = str.encode(text)
  object_write_result = client.put_object(bucket_name, object_name,
                            io.BytesIO(text_as_bytes), len(text_as_bytes))
  return object_write_result

batch_data_bucket_name = 'batch-inference'
count = 0

json_list = []
with open('emotions-test.jsonl', 'r') as json_file:
  json_list = list(json_file)

for json_str in json_list:
  object_name = f'{uuid.uuid4()}.json'
  put_text(batch_data_bucket_name, object_name, json_str)
  count += 1

print(f'{count} 个对象已上传。)

2000 个对象已上传。

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device

model_check_point = 'distilbert-base-uncased'
num_labels = 6
model = AutoModelForSequenceClassification.from_pretrained(model_check_point,

                     num_labels=num_labels).to(device)
tokenizer = AutoTokenizer.from_pretrained(model_check_point)

emotions.reset_format()
def tokenize(batch) -> BatchEncoding:
  out = tokenizer(batch['text'], padding=True, truncation=True, return_tensors='pt')
  return out

emotions_encoded = emotions.map(tokenize, batched=True, batch_size=None)

from sklearn.metrics import accuracy_score, f1_score

def compute_metrics(pred):
  labels = pred.label_ids
  preds = pred.predictions.argmax(-1)
  f1 = f1_score(labels, preds, average='weighted')
  accuracy = accuracy_score(labels, preds)
  return {'accuracy': accuracy, 'f1': f1}

from transformers import TrainingArguments

batch_size = 64
logging_steps = len(emotions_encoded['train']) // batch_size
model_name = f'{model_check_point}-finetuned-emotion'

training_args = TrainingArguments(output_dir=model_name,
                      num_train_epochs=2,
                      learning_rate=2e-5,
                      per_device_train_batch_size=batch_size,
                      weight_decay=0.01,
                      evaluation_strategy='epoch',
                      disable_tqdm=False,
                      logging_steps=logging_steps,
                      push_to_hub=False,
                      log_level='error')

从 transformers 导入 Trainer

trainer = Trainer(model=model,
                  args=training_args,
                  compute_metrics=compute_metrics,
                  train_dataset=emotions_encoded['train'],
                  eval_dataset=emotions_encoded['validation'],
                  tokenizer=tokenizer)

trainer.train()

{'eval_loss': 0.2137589156627655, 'eval_accuracy': 0.924, 'eval_f1': 0.9240498809080812, 'eval_runtime': 2.5206, 'eval_samples_per_second': 793.448, 'eval_steps_per_second': 99.181, 'epoch': 2.0}

def get_object_list(bucket_name: str, prefix: str=None) -> List[str]:
  '''
  Gets a list of objects from a bucket.
  '''
  # Load the credentials and connection information.
  with open('credentials.json') as f:
    credentials = json.load(f)

  # Get data of an object.
  try:
    # Create client with access and secret key
    client = Minio(credentials['url'], # host.docker.internal
                  credentials['accessKey'],
                  credentials['secretKey'],
                  secure=False)

    object_list = []
    objects = client.list_objects(bucket_name, prefix=prefix, recursive=True)
    for obj in objects:
      object_list.append(obj.object_name)
    except S3Error as s3_err:
      raise s3_err
    except Exception as err:
      raise err

  return object_list

def get_new_data(bucket_name: str, file_path: str) -> None:
  object_list = get_object_list(bucket_name)

  json_list = []
  for obj in object_list:
  json_str = get_text(bucket_name, obj)
  json_list.append(json_str)

  with open(file_path, 'w') as f:
    for item in json_list:
      f.write(item)

data_file = 'batch_data.jsonl'
get_new_data(batch_data_bucket_name, data_file)
new_data = load_dataset('json', data_files={'batch_data': data_file})
new_data_encoded = new_data.map(tokenize, batched=True, batch_size=None)

predictions = trainer.predict(new_data_encoded['batch_data'])
predictions.metrics

{'test_loss': 0.22074727714061737, 'test_accuracy': 0.9115, 'test_f1': 0.9116612061453423, 'test_runtime': 2.4571, 'test_samples_per_second': 813.959, 'test_steps_per_second': 101.745}