func hitTest(_ point: CGPoint,
   withEvent event: UIEvent?) -> UIView? // that's for handling the case of multiple custom subviews on your view rather than for evaluating if it's your view to handle the touch

func pointInside(_ point: CGPoint,
       withEvent event: UIEvent?) -> Bool // way more preferable than hit test in your case!!!

func alphaFromPoint(point: CGPoint) -> CGFloat {
   var pixel: [UInt8] = [0, 0, 0, 0]
   let colorSpace = CGColorSpaceCreateDeviceRGB();
   let alphaInfo : CGBitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.PremultipliedLast.rawValue)
   let context = CGBitmapContextCreate(&pixel, 1, 1, 8, 4, colorSpace, alphaInfo.rawValue) //need add .rawValue to alphaInfo

   CGContextTranslateCTM(context, -point.x, -point.y);

   self.layer.renderInContext(context!)

   let floatAlpha = CGFloat(pixel[3])
   return floatAlpha

extension UIButton{

    open override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
        if let touch = event!.touches(for: self)?.first {
            let location = touch.location(in: self)
            if alphaFromPoint(point: location) == 0 {
                self.cancelTracking(with: nil)
                print("cancelled!")
            } else{
                super.touchesBegan(touches, with: event)
            }
        }
    }

    func alphaFromPoint(point: CGPoint) -> CGFloat {
        var pixel: [UInt8] = [0, 0, 0, 0]
        let colorSpace = CGColorSpaceCreateDeviceRGB();
        let alphaInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue)
        let context = CGContext(data: &pixel, width: 1, height: 1, bitsPerComponent: 8, bytesPerRow: 4, space: colorSpace, bitmapInfo: alphaInfo.rawValue)

        context!.translateBy(x: -point.x, y: -point.y)
        self.layer.render(in: context!)

        let floatAlpha = CGFloat(pixel[3])
        return floatAlpha
    }
}