localPlayer = game:getService("Players").localPlayer
playerGui = localPlayer:waitForChild("PlayerGui")
mouse = localPlayer:getMouse()

function checkMouseEntered(rect)
    local mousePosition = CFrame.new(mouse.X, mouse.Y, 0)
    local rectPosition = CFrame.new(rect.Position.X.Offset+rect.Size.X.Offset/2, rect.Position.Y.Offset+rect.Size.Y.Offset/2, 0)*CFrame.Angles(0, 0, math.rad(rect.Rotation))
    local relativeMouse = mousePosition:toObjectSpace(rectPosition)
    if relativeMouse.X > 0 and relativeMouse.X < rect.Size.X.Offset and relativeMouse.Y > 0 and relativeMouse.Y < rect.Size.Y.Offset then
        return true, UDim2.new(0, mousePosition.X, 0, mousePosition.Y)
    else
        return false, UDim2.new(0, mousePosition.X, 0, mousePosition.Y)
    end
end

thing = Instance.new("ScreenGui", playerGui)

rect = Instance.new("Frame", thing)
rect.Size = UDim2.new(0, 600, 0, 300)
rect.Position = UDim2.new(0, 500, 0, 400)
rect.Rotation = 35

mouse.Move:connect(function()
    local value, pos = checkMouseEntered(rect)
    print(value, pos)
    if value == false then
        local newThing = Instance.new("Frame", thing)
        newThing.Size = UDim2.new(0, 5, 0, 5)
        newThing.Position = pos
        newThing.BackgroundColor3 = Color3.new(1, 0, 0)
    end
end)