Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import os
- from transformers import (
- AutoTokenizer,
- AutoModel,
- AutoModelForMaskedLM,
- AutoModelForSequenceClassification,
- AutoModelForTokenClassification,
- )
- MODELS_DIR = "models"
- MODELS = [
- # Дообучение + заполнение маски (лёгкая универсальная модель)
- ("cointegrated/rubert-tiny2",
- AutoModelForSequenceClassification, "классификация, дообучение"),
- ("cointegrated/rubert-tiny2",
- AutoModelForMaskedLM, "заполнение маски [MASK]"),
- ("cointegrated/rubert-tiny2",
- AutoModel, "эмбеддинги, сравнение текстов"),
- # Тональность (готовая, обучать не надо)
- ("blanchefort/rubert-base-cased-sentiment",
- AutoModelForSequenceClassification, "тональность текста"),
- # NER (готовая, обучать не надо)
- ("Davlan/bert-base-multilingual-cased-ner-hrl",
- AutoModelForTokenClassification, "NER: имена, места, организации"),
- ]
- for name, model_class, description in MODELS:
- suffix = model_class.__name__.replace("AutoModel", "").lower()
- folder = name.replace("/", "_") + (f"_{suffix}" if suffix else "")
- local_path = os.path.join(MODELS_DIR, folder)
- if os.path.exists(local_path):
- print(f" ✓ Уже есть: {name} ({description})")
- continue
- print(f" ⏳ Скачиваю: {name}")
- print(f" Задача: {description}")
- tokenizer = AutoTokenizer.from_pretrained(name)
- if model_class == AutoModelForSequenceClassification:
- model = model_class.from_pretrained(name, num_labels=2,
- ignore_mismatched_sizes=True)
- else:
- model = model_class.from_pretrained(name)
- os.makedirs(local_path, exist_ok=True)
- tokenizer.save_pretrained(local_path)
- model.save_pretrained(local_path)
- print(f" ✓ Сохранена в: {local_path}\n")
- print("\n" + "═" * 50)
- print("Готово! Скачанные модели:\n")
- for d in sorted(os.listdir(MODELS_DIR)):
- path = os.path.join(MODELS_DIR, d)
- size = sum(
- os.path.getsize(os.path.join(path, f))
- for f in os.listdir(path)
- ) / 1024 / 1024
- print(f" {d:55s} {size:>7.0f} МБ")
Advertisement
Add Comment
Please, Sign In to add comment