public class TextBoxViewModel : ViewModelBase
{
    private string _text;

    public TextBoxViewModel()
    {
        ShowCalculatorCommand = new RelayCommand(ShowCalculator);
    }

    public string Text
    {
        get { return _text; }
        set { Set(ref _text, value); }
    }

    public ICommand ShowCalculatorCommand { get; }

    private void ShowCalculator()
    {
        MessengerInstance.Send(new NotificationMessage<string>(Text, "ShowCalculator"));

        // Слушаем нотификацию "SubmitCalculator".
        MessengerInstance.Register<NotificationMessage<string>>(this, message =>
        {
            if (message.Notification == "SubmitCalculator")
            {
                Text = message.Content;
            }
        });

        // Слушаем нотификацию "CloseCalculator".
        MessengerInstance.Register<NotificationMessage<string>>(this, message =>
        {
            if (message.Notification == "CloseCalculator")
            {
                // Калькулятор закрыт - отписываемся от нотификаций.
                MessengerInstance.Unregister(this);
            }
        });
    }
}

// В обработчике события Loaded главного окна.
Messenger.Default.Register<NotificationMessage<string>>(this, message =>
{
    if (message.Notification == "ShowCalculator")
    {
        var window = new CalculatorWindow
        {
            DataContext = new CalculatorViewModel { Text = message.Content }
        };

        window.Show();
    }
});

// В обработчике события Loaded окна с калькулятором.
Messenger.Default.Register<NotificationMessage>(this, message =>
{
    if (message.Notification == "CloseCalculator")
    {
        Close();
    }
});

public class CalculatorViewModel : ViewModelBase
{
    private string _text;

    public CalculatorViewModel()
    {
        OkCommand = new RelayCommand(Ok);
        CancelCommand = new RelayCommand(Cancel);
    }

    public string Text
    {
        get { return _text; }
        set { Set(ref _text, value); }
    }

    public ICommand OkCommand { get; }
    public ICommand CancelCommand { get; }

    private void Ok()
    {
        MessengerInstance.Send(new NotificationMessage<string>(Text, "SubmitCalculator"));
        MessengerInstance.Send(new NotificationMessage("CloseCalculator"));
    }

    private void Cancel()
    {
        MessengerInstance.Send(new NotificationMessage("CloseCalculator"));
    }
}